diff --git a/.adonisjs/client/manifest.d.ts b/.adonisjs/client/manifest.d.ts index 27cbe6d..852b3d3 100644 --- a/.adonisjs/client/manifest.d.ts +++ b/.adonisjs/client/manifest.d.ts @@ -5,7 +5,9 @@ /// /// +/// /// +/// /// /// /// diff --git a/.adonisjs/server/controllers.ts b/.adonisjs/server/controllers.ts index 202eb41..ed3a28e 100644 --- a/.adonisjs/server/controllers.ts +++ b/.adonisjs/server/controllers.ts @@ -6,5 +6,7 @@ export const controllers = { AccessTokens: () => import('#controllers/access_tokens_controller'), NewAccount: () => import('#controllers/new_account_controller'), + Newznab: () => import('#controllers/newznab_controller'), Profile: () => import('#controllers/profile_controller'), + Releases: () => import('#controllers/releases_controller'), } diff --git a/.adonisjs/server/routes.d.ts b/.adonisjs/server/routes.d.ts index 9aa155f..37d0ca6 100644 --- a/.adonisjs/server/routes.d.ts +++ b/.adonisjs/server/routes.d.ts @@ -4,15 +4,27 @@ type ParamValue = string | number | bigint | boolean export type ScannedRoutes = { ALL: { + 'releases.index': { paramsTuple?: []; params?: {} } + 'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'releases.stats': { paramsTuple?: []; params?: {} } 'auth.new_account.store': { paramsTuple?: []; params?: {} } 'auth.access_tokens.store': { paramsTuple?: []; params?: {} } 'profile.profile.show': { paramsTuple?: []; params?: {} } 'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} } } GET: { + 'releases.index': { paramsTuple?: []; params?: {} } + 'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'releases.stats': { paramsTuple?: []; params?: {} } 'profile.profile.show': { paramsTuple?: []; params?: {} } } HEAD: { + 'releases.index': { paramsTuple?: []; params?: {} } + 'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } + 'releases.stats': { paramsTuple?: []; params?: {} } 'profile.profile.show': { paramsTuple?: []; params?: {} } } POST: { diff --git a/.adonisjs/server/routes.json b/.adonisjs/server/routes.json new file mode 100644 index 0000000..b0765b6 --- /dev/null +++ b/.adonisjs/server/routes.json @@ -0,0 +1 @@ +{"root":[{"domain":"root","pattern":"/api/v1/releases","matchers":{},"tokens":[{"old":"/api/v1/releases","type":0,"val":"api","end":""},{"old":"/api/v1/releases","type":0,"val":"v1","end":""},{"old":"/api/v1/releases","type":0,"val":"releases","end":""}],"meta":{"params":[]},"name":"releases.index","handler":{"method":"index","reference":[null,"index"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id","type":1,"val":"id","end":"","matcher":{}}],"meta":{"params":["id"]},"name":"releases.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id/nzb","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id/nzb","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id/nzb","type":1,"val":"id","end":"","matcher":{}},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"nzb","end":""}],"meta":{"params":["id"]},"name":"releases.nzb","handler":{"method":"nzb","reference":[null,"nzb"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/stats","matchers":{},"tokens":[{"old":"/api/v1/stats","type":0,"val":"api","end":""},{"old":"/api/v1/stats","type":0,"val":"v1","end":""},{"old":"/api/v1/stats","type":0,"val":"stats","end":""}],"meta":{"params":[]},"name":"releases.stats","handler":{"method":"stats","reference":[null,"stats"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/signup","matchers":{},"tokens":[{"old":"/api/v1/auth/signup","type":0,"val":"api","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"signup","end":""}],"meta":{"params":[]},"name":"auth.new_account.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/new_account_controller')","name":"NewAccount"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/login","matchers":{},"tokens":[{"old":"/api/v1/auth/login","type":0,"val":"api","end":""},{"old":"/api/v1/auth/login","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/login","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/login","type":0,"val":"login","end":""}],"meta":{"params":[]},"name":"auth.access_tokens.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/profile","matchers":{},"tokens":[{"old":"/api/v1/account/profile","type":0,"val":"api","end":""},{"old":"/api/v1/account/profile","type":0,"val":"v1","end":""},{"old":"/api/v1/account/profile","type":0,"val":"account","end":""},{"old":"/api/v1/account/profile","type":0,"val":"profile","end":""}],"meta":{"params":[]},"name":"profile.profile.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/profile_controller')","name":"Profile"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/logout","matchers":{},"tokens":[{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],"meta":{"params":[]},"name":"profile.access_tokens.destroy","handler":{"method":"destroy","reference":[null,"destroy"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}}]} \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..864de83 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +node_modules +build +tmp +debug +.git +.idea +.vscode +.env +.env.* +!.env.example +*.log diff --git a/.env.example b/.env.example index 79a6cf8..f153372 100644 --- a/.env.example +++ b/.env.example @@ -15,4 +15,6 @@ SESSION_DRIVER=cookie #-------------------------------------------------------------------- # CORS (configure allowed origins for API access) #-------------------------------------------------------------------- -# CORS_ORIGIN=http://localhost:5173,http://localhost:3000 \ No newline at end of file +# CORS_ORIGIN=http://localhost:5173,http://localhost:3000 +# Newznab API key (/api?apikey=...); leave empty to allow access without a key +NEWZNAB_API_KEY= diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 0000000..7858245 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +24.16 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..3b1dab0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +# Node is pinned: on 24.21, jsonschema 1.5.0 (used by Ace to validate commands) +# throws "Invalid URL" on $ref, so no ace command starts. See LOG.md. +ARG NODE_VERSION=24.16 + +# Build stage: dev dependencies, TypeScript compile, native addons. +# The full node image ships python3/make/g++, needed to compile yencode. +FROM node:${NODE_VERSION}-bookworm AS build +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci +COPY . . +RUN node ace build \ + && cd build \ + && npm ci --omit=dev + +# Runtime stage: compiled app with production dependencies only. +FROM node:${NODE_VERSION}-bookworm-slim +ENV NODE_ENV=production +# bsdtar (libarchive) unpacks compressed RAR entries, e.g. NZBs posted inside RARs. +RUN apt-get update \ + && apt-get install -y --no-install-recommends libarchive-tools \ + && rm -rf /var/lib/apt/lists/* +WORKDIR /app +COPY --from=build /app/build ./ +USER node +# Every service picks its own command, e.g. `node ace worker:fetch`. +CMD ["node", "ace", "list"] diff --git a/adonisrc.ts b/adonisrc.ts index a96390d..832e211 100644 --- a/adonisrc.ts +++ b/adonisrc.ts @@ -53,6 +53,7 @@ export default defineConfig({ () => import('@adonisjs/cors/cors_provider'), () => import('@adonisjs/auth/auth_provider'), () => import('#providers/api_provider'), + () => import('@adonisjs/static/static_provider') ], /* @@ -103,7 +104,10 @@ export default defineConfig({ | the production build. | */ - metaFiles: [], + metaFiles: [{ + pattern: 'public/**', + reloadServer: false, + }], hooks: { init: [ diff --git a/app/controllers/newznab_controller.ts b/app/controllers/newznab_controller.ts new file mode 100644 index 0000000..d78d020 --- /dev/null +++ b/app/controllers/newznab_controller.ts @@ -0,0 +1,121 @@ +import type { HttpContext } from '@adonisjs/core/http' +import env from '#start/env' +import SearchService from '#services/SearchService' +import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index' + +const LIMITS = { max: 100, default: 50 } + +/** Search functions and the category each defaults to when `cat` is absent. */ +const SEARCH_FUNCTIONS: Record = { + 'search': undefined, + 'tvsearch': 5000, + 'tv-search': 5000, + 'movie': 2000, + 'movie-search': 2000, + 'music': 3000, + 'audio': 3000, + 'book': 7000, +} + +/** The configured key, if any (a Secret from .env, or a plain string set at runtime). */ +function configuredApiKey(): string | undefined { + const value: any = env.get('NEWZNAB_API_KEY') + const key = typeof value === 'string' ? value : value?.release() + return key || undefined +} + +function int(value: unknown): number | undefined { + return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : undefined +} + +/** + * Newznab API (CONCEPT.md D23) for Prowlarr, Sonarr, Radarr and the like: + * GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=… + */ +export default class NewznabController { + async handle({ request, response }: HttpContext) { + const qs = request.qs() + const t = String(qs.t ?? '') + const server = this.server(request) + + const xml = (body: string, type = 'application/xml') => { + response.header('Content-Type', `${type}; charset=utf-8`) + return response.send(body) + } + + const apiKey = configuredApiKey() + if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials')) + + if (t === 'caps') return xml(capsXml(server, LIMITS)) + + if (t === 'get') { + const id = int(qs.id) + if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)')) + const nzb = await SearchService.nzb(id) + if (!nzb) return xml(errorXml(300, 'No such item')) + const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb` + response.header('Content-Disposition', `attachment; filename="${filename}"`) + return xml(nzb.xml, 'application/x-nzb') + } + + if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`)) + + const requested = String(qs.cat ?? '') + .split(',') + .map((c) => int(c.trim())) + .filter((c): c is number => c !== undefined) + const fallback = SEARCH_FUNCTIONS[t] + const categories = expandCategories(requested.length ? requested : fallback ? [fallback] : []) + const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max) + const offset = int(qs.offset) ?? 0 + + // ID-only lookups (tvdbid, imdbid, …) are not supported: caps says so, but + // answer with an empty result rather than everything. + const q = typeof qs.q === 'string' ? qs.q.trim() : '' + const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p]) + + const result = idOnly + ? { total: 0, releases: [] } + : await SearchService.search({ + q, + categories, + readable: true, + notIncomplete: true, + hideContainers: true, + maxAgeDays: int(qs.maxage), + season: int(qs.season), + episode: int(qs.season) !== undefined ? int(qs.ep) : undefined, + sort: 'newest', + perPage: limit, + offset, + }) + + const items = result.releases.map((r: any) => { + const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name) + return { + id: r.id, + title: r.name, + category: r.category ?? 8010, + size: r.size, + files: r.fileCount, + postedAt: r.postedAt ? new Date(r.postedAt) : null, + poster: r.poster, + groups: r.groups ?? [], + password: Boolean(r.encrypted), + season: tv ? `S${tv[1].padStart(2, '0')}` : undefined, + episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined, + } + }) + return xml(resultsXml(server, items, offset, result.total), 'application/rss+xml') + } + + private server(request: HttpContext['request']): NewznabServer { + const proto = request.header('x-forwarded-proto') ?? 'http' + const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost' + return { + title: 'Usenet Indexer', + baseUrl: `${proto}://${host}`, + apiKey: configuredApiKey(), + } + } +} diff --git a/app/controllers/releases_controller.ts b/app/controllers/releases_controller.ts new file mode 100644 index 0000000..b777061 --- /dev/null +++ b/app/controllers/releases_controller.ts @@ -0,0 +1,44 @@ +import type { HttpContext } from '@adonisjs/core/http' +import SearchService, { type SortOrder } from '#services/SearchService' + +const flag = (value: unknown) => value === '1' || value === 'true' || value === true + +export default class ReleasesController { + /** GET /api/v1/releases?q=&readable=1&complete=1&hide_encrypted=1&source=&poster=&sort=&page=&per_page= */ + async index({ request }: HttpContext) { + const qs = request.qs() + return SearchService.search({ + q: typeof qs.q === 'string' ? qs.q : undefined, + readable: flag(qs.readable), + complete: flag(qs.complete), + hideEncrypted: flag(qs.hide_encrypted), + includePar2Only: flag(qs.par2_only), + source: typeof qs.source === 'string' && qs.source ? qs.source : undefined, + poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined, + sort: qs.sort as SortOrder | undefined, + page: Number(qs.page) || 1, + perPage: Number(qs.per_page) || 50, + }) + } + + /** GET /api/v1/releases/:id */ + async show({ params, response }: HttpContext) { + const release = await SearchService.release(Number(params.id)) + return release ?? response.notFound({ error: 'Release not found' }) + } + + /** GET /api/v1/releases/:id/nzb */ + async nzb({ params, response }: HttpContext) { + const nzb = await SearchService.nzb(Number(params.id)) + if (!nzb) return response.notFound({ error: 'Release not found' }) + const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb` + response.header('Content-Type', 'application/x-nzb') + response.header('Content-Disposition', `attachment; filename="${filename}"`) + return response.send(nzb.xml) + } + + /** GET /api/v1/stats */ + async stats() { + return SearchService.stats() + } +} diff --git a/app/models/file.ts b/app/models/file.ts index 331ca9d..d0a8231 100644 --- a/app/models/file.ts +++ b/app/models/file.ts @@ -6,6 +6,9 @@ export default class File extends BaseModel { @column({ isPrimary: true }) declare id: number + @column({ columnName: 'file_key' }) + declare fileKey: string | null + @column() declare filename: string diff --git a/app/models/group.ts b/app/models/group.ts index f1cab96..b9ecd05 100644 --- a/app/models/group.ts +++ b/app/models/group.ts @@ -12,9 +12,18 @@ export default class Group extends BaseModel { @column() declare active: boolean + /** Every article up to here was fetched successfully (see FetchRangeService). */ @column({ columnName: 'last_indexed_id' }) declare lastIndexedId: bigint + /** Download bodies of obfuscated articles to identify them (bandwidth-heavy). */ + @column({ columnName: 'fetch_bodies' }) + declare fetchBodies: boolean + + /** How far the scheduler has queued fetch jobs. */ + @column({ columnName: 'last_queued_id' }) + declare lastQueuedId: bigint + @column.dateTime({ autoCreate: true }) declare createdAt: DateTime diff --git a/app/services/ArchiveService.ts b/app/services/ArchiveService.ts new file mode 100644 index 0000000..980df7b --- /dev/null +++ b/app/services/ArchiveService.ts @@ -0,0 +1,160 @@ +import { execFile } from 'node:child_process' +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { promisify } from 'node:util' +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import IngestService from '#services/IngestService' +import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index' +import { fileStem, rarReleaseIdentity } from '#lib/release/index' + +/** Stored per file and release; a listing of thousands of files is not useful. */ +const MAX_ENTRIES = 500 + +export interface ArchiveResult { + info: RarInfo + releaseId?: number + releaseRenamed?: string + grouped?: boolean +} + +/** + * Records what a RAR volume's headers say: the listing on the file, the + * release it belongs to (from the shared directory or continued entry, when + * the file has none yet), and for first volumes the contents and password + * flag on the release. See CONCEPT.md (D19, D21). + */ +class ArchiveService { + async record(file: any, binary: any, info: RarInfo): Promise { + const stored = { ...info, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) } + await db.from('files').where('id', file.id).update({ archive: JSON.stringify(stored) }) + + let releaseId = file.release_id ?? binary?.release_id ?? null + let grouped = false + const identity = rarReleaseIdentity(info.entries) + if (!releaseId && identity) { + const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [ + { + release: identity, + poster: binary?.poster ?? file.poster, + weakIdentity: true, + groups: binary?.groups ?? file.groups ?? [], + postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined, + }, + ]) + releaseId = ids.get(identity.key)! + grouped = true + if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId }) + await db.from('files').where('id', file.id).update({ release_id: releaseId }) + } + if (!releaseId) return { info } + + const release = await db.from('releases').where('id', releaseId).first() + const update: Record = {} + if (info.firstVolume !== false) { + update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted) + if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries) + } else if (info.headersEncrypted) { + update.encrypted = true + } + const better = releaseNameFromContents(info) + const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined + if (renamed) update.name = renamed + if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update) + + if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`) + return { info, releaseId, releaseRenamed: renamed, grouped } + } +} + +const run = promisify(execFile) + +/** Tools that print one archive entry to stdout, tried in order. */ +const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [ + ['bsdtar', (archive, entry) => ['-xOf', archive, entry]], + ['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]], +] + +/** + * The unpacked data of one entry of a complete (single-volume) archive: + * copied out when stored, else unpacked with bsdtar (libarchive, in the + * Docker image) or unrar. Null when that is not possible. + */ +export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise { + const stored = storedEntryData(archive, entry) + if (stored) return stored + if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null + if (entry.size !== null && entry.size > maxBytes) return null + + const dir = await mkdtemp(path.join(tmpdir(), 'rar-')) + try { + const file = path.join(dir, 'archive.rar') + await writeFile(file, archive) + for (const [tool, args] of EXTRACTORS) { + try { + const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 }) + if (stdout.length > 0) return stdout + } catch (error: any) { + if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`) + } + } + return null + } finally { + await rm(dir, { recursive: true, force: true }) + } +} + +/** + * A release name from an archive listing: the top-level directory shared by + * all entries when it reads like a release name, otherwise the stem of the + * largest entry with a readable name (e.g. an archive holding just + * `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped. + */ +export function releaseNameFromContents(info: Pick): string | undefined { + const entries = info.entries.filter((e) => !e.directory) + const tops = new Set(info.entries.map((e) => e.name.split(/[\\/]/)[0])) + if (tops.size === 1 && info.entries.some((e) => /[\\/]/.test(e.name))) { + const [top] = tops + if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top + } + const candidates = entries + .map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 })) + .filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem)) + .filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem)) + .sort((a, b) => b.size - a.size) + return candidates[0]?.stem +} + +/** + * Renames random-named releases from their stored archive listing (no + * downloads). Used by `release:rename`. + */ +export async function renameFromStoredContents(): Promise { + const releases = await db + .from('releases') + .whereNotNull('contents') + .whereRaw("jsonb_array_length(contents) > 0") + .select('id', 'name', 'source', 'contents') + let renamed = 0 + for (const release of releases) { + if (!looksRandom(release.name) && release.source !== 'post') continue + const better = releaseNameFromContents({ entries: release.contents }) + if (!better || better === release.name) continue + await db.from('releases').where('id', release.id).update({ name: better }) + renamed++ + } + return renamed +} + +/** Obfuscated names: one long token of mixed letters and digits, no separators. */ +export function looksRandom(name: string): boolean { + // An unknown short extension (e.g. ".bin") says nothing about the name. + name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '') + if (/^[0-9a-f]{16,}$/i.test(name)) return true + if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false + // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk). + return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6 +} + +export default new ArchiveService() diff --git a/app/services/ContentService.ts b/app/services/ContentService.ts new file mode 100644 index 0000000..9ed221e --- /dev/null +++ b/app/services/ContentService.ts @@ -0,0 +1,187 @@ +import { createHash } from 'node:crypto' +import db from '@adonisjs/lucid/services/db' +import NntpService from '#services/NntpService' +import Par2Service, { type Par2Result } from '#services/Par2Service' +import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService' +import NameService from '#services/NameService' +import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index' +import NzbImportService, { type NzbImportResult } from '#services/NzbImportService' +import { sniffContentType, type ContentType } from '#lib/content/index' +import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index' +import { fileStem } from '#lib/release/index' +import { looksLikeFilename } from '#lib/subject/index' + +const FIRST_CHUNK = 16 * 1024 + +/** + * MD5 of the first 16 KiB of a file, as par2 records it, from the body of + * the file's first part. Undefined when that part is shorter than 16 KiB but + * the file is not (the chunk would span two parts). + */ +export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined { + return chunkMd5(decodeSegment(body).data, fileSize) +} + +function chunkMd5(data: Buffer, fileSize: number): string | undefined { + if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined + return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex') +} + +export interface ContentResult { + par2?: Par2Result + archive?: ArchiveResult + named: number + contentType: ContentType | null + nzb?: NzbImportResult + /** NZBs read from inside an archive. */ + archiveNzbs?: NzbImportResult[] +} + +/** + * Takes names and groupings from file contents (CONCEPT.md D21). For a + * completed file with a random name, one article (the first) is enough: + * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry, + * - par2 files are recognised by content and name every file of their set, + * - RAR headers give the release grouping, name and contents. + */ +class ContentService { + /** Random or missing names, and named first RAR volumes (for contents and password flag). */ + isCandidate(name: string) { + if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true + if (/\.nzb$/i.test(name)) return true + return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name)) + } + + /** Downloads a whole (small) file whose first segment is already decoded. */ + private async readWhole(first: Buffer, messageIds: Record): Promise { + const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b)) + const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0) + if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null + if (parts.length === 1) return first + + let assembled: YencFile | undefined + const conn = await NntpService.acquire() + let failed = false + try { + for (const [, part] of parts) { + const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true }) + assembled ??= YencFile.fromSegment(segment) + assembled.add(segment) + } + } catch (error) { + failed = true + throw error + } finally { + NntpService.release(conn, failed) + } + return assembled!.getBuffer() + } + + /** + * Downloads a small archive that lists NZBs and imports each of them + * (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated + * uploads by other posters. + */ + private async importArchiveNzbs(file: any, first: Buffer): Promise { + const archive = await this.readWhole(first, file.message_ids) + const info = archive && parseRarHeaders(archive) + if (!archive || !info) return [] + const entries = info.entries.filter(isNzbEntry) + const results: NzbImportResult[] = [] + for (const entry of entries) { + const xml = await extractEntry(archive, entry) + if (!xml) continue + const imported = await NzbImportService.import(file, xml.toString('utf8'), { + entryName: entry.name, + moveContainer: entries.length === 1, + }) + if (imported) results.push(imported) + } + return results + } + + async inspect(fileId: number): Promise { + const file = await db.from('files').where('id', fileId).first() + const binary = await db.from('binaries').where('file_id', fileId).first() + if (!file || !this.isCandidate(file.filename)) return null + const first = file.message_ids?.['1'] + if (!first?.id) return null + + const conn = await NntpService.acquire() + let failed = false + let body: Buffer + try { + body = await NntpService.fetchBody(conn, first.id) + } catch (error) { + failed = true + throw error + } finally { + NntpService.release(conn, failed) + } + // Some tools declare a size that leaves out a header (nzbe); the CRC still has to match. + let segment + try { + segment = decodeSegment(body) + } catch (error) { + if (!(error instanceof YencError) || error instanceof YencCrcError) throw error + segment = decodeSegment(body, { lenientSize: true }) + } + const size = segment.meta.begin.size + const contentType = sniffContentType(segment.data) + await db.from('files').where('id', fileId).update({ content_type: contentType }) + + // Fingerprint, so par2 sets (now or later) can name this file exactly. + if (binary) { + await db + .from('binaries') + .where('id', binary.id) + .update({ + size: db.raw('coalesce(size, ?)', [size]), + md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]), + }) + } + + const result: ContentResult = { named: 0, contentType } + if (contentType === 'nzbe' || contentType === 'rclone') { + // An encrypted NZB, or rclone crypt (names and data encrypted): nothing + // to read, and useless without the key. + const releaseId = file.release_id ?? binary?.release_id + if (releaseId) { + await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true }) + } + return result + } + if (contentType === 'nzb') { + const xml = await this.readWhole(segment.data, file.message_ids) + if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined + return result + } + const set = await Par2Service.read(segment.data, file.message_ids) + if (set && binary) { + result.par2 = await Par2Service.attach(binary, set) + } else { + const info = parseRarHeaders(segment.data) + if (info) { + result.archive = await ArchiveService.record(file, binary, info) + if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data) + } + } + if (binary) result.named = await NameService.resolve([binary.id]) + return result + } +} + +/** NZBs that can be read from a single, unencrypted archive. */ +function hasNzbEntries(info: RarInfo): boolean { + return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry) +} + +function isNzbEntry(entry: RarEntry): boolean { + return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory +} + +/** Small posted files (NZBs) that are read completely. */ +const WHOLE_MAX_PARTS = 40 +const WHOLE_MAX_BYTES = 25 * 1024 * 1024 + +export default new ContentService() diff --git a/app/services/DeferredService.ts b/app/services/DeferredService.ts new file mode 100644 index 0000000..2e09d94 --- /dev/null +++ b/app/services/DeferredService.ts @@ -0,0 +1,264 @@ +import { createHash } from 'node:crypto' +import db from '@adonisjs/lucid/services/db' +import indexerConfig from '#config/indexer' +import NntpService from '#services/NntpService' +import QueueService from '#services/QueueService' +import { type OverviewHeader } from '#services/IngestService' +import { decodeSegment, inferTotalParts, parseYencMeta } from '#lib/yenc/index' +import { sniffContentType } from '#lib/content/index' +import { parseRarHeaders } from '#lib/rar/index' + +/** Expanding re-reads the file's article range with XOVER; beyond this it is too costly. */ +const EXPAND_MAX_RANGE = 200_000 +/** A claimed row whose worker died is taken again after this long. */ +const RECLAIM_AFTER = '1 hour' + +export interface DeferredArticle { + articleNumber: number + header: OverviewHeader +} + +/** File name from a deferred uploader's subject: `[i/M] "name" yEnc`, no part counter. */ +export function deferredName(subject: string | undefined): string | null { + return /"([^"]+)"/.exec(subject ?? '')?.[1]?.trim() || null +} + +export function isDeferred(header: OverviewHeader): boolean { + const id = header['message-id'] ?? '' + return indexerConfig.deferBodyMessageIds.some((re) => re.test(id)) +} + +/** A recorded range never spans more articles than this (one XOVER to expand it). */ +const RANGE_MAX_SPAN = 50_000 + +export function bodyJob(header: OverviewHeader, group: string, articleNumber: number | bigint) { + return { + name: 'process-body', + data: { header, group, articleNumber: Number(articleNumber) }, + // By message-id: an article crossposted to several indexed groups is downloaded once. + opts: { jobId: `body-${createHash('sha1').update(header['message-id'] ?? '').digest('hex')}` }, + } +} + +/** + * Deferred uploads (CONCEPT.md D28). Their articles skip the body queue and + * are recorded here per file, from the headers alone. `worker:deferred` + * later downloads one body per file for its size, part count and content + * type, and sends small files (NZBs in RARs, par2 index files) to the body + * queue completely. + */ +class DeferredService { + /** + * Records one group's deferred articles: per file when the subject names + * the file, otherwise only as article ranges. Returns the number of files. + */ + async record(groupName: string, articles: DeferredArticle[]): Promise { + const named = articles.filter((a) => deferredName(a.header.subject) !== null) + await this.recordRanges(groupName, articles.filter((a) => deferredName(a.header.subject) === null)) + return this.recordFiles(groupName, named) + } + + private async recordRanges(groupName: string, articles: DeferredArticle[]) { + if (articles.length === 0) return + const sorted = [...articles].sort((a, b) => Number(a.articleNumber) - Number(b.articleNumber)) + const now = new Date() + const rows: Record[] = [] + for (const article of sorted) { + const n = Number(article.articleNumber) + const bytes = Number.parseInt(article.header[':bytes'] ?? '0', 10) || 0 + const last = rows.at(-1) + if (last && n - (last.first_article as number) <= RANGE_MAX_SPAN) { + last.last_article = n + ;(last.article_count as number)++ + last.bytes = (last.bytes as number) + bytes + } else { + rows.push({ group_name: groupName, first_article: n, last_article: n, article_count: 1, bytes, created_at: now }) + } + } + await db.table('deferred_ranges').multiInsert(rows) + } + + private async recordFiles(groupName: string, articles: DeferredArticle[]): Promise { + const files = new Map() + for (const { articleNumber, header } of articles) { + const name = deferredName(header.subject)! + const n = Number(articleNumber) + const bytes = Number.parseInt(header[':bytes'] ?? '0', 10) || 0 + const date = header.date ? new Date(header.date) : null + const postedAt = date && !Number.isNaN(date.getTime()) ? date : null + const file = files.get(name) + if (!file) { + files.set(name, { + name, subject: header.subject ?? '', poster: header.from ?? '', first: n, + firstId: (header['message-id'] ?? '').replace(/^<|>$/g, ''), last: n, count: 1, bytes, postedAt, + }) + continue + } + if (n < file.first) { + file.first = n + file.firstId = (header['message-id'] ?? '').replace(/^<|>$/g, '') + } + file.last = Math.max(file.last, n) + file.count++ + file.bytes += bytes + if (postedAt && (!file.postedAt || postedAt < file.postedAt)) file.postedAt = postedAt + } + if (files.size === 0) return 0 + + const now = new Date() + const rows = [...files.values()].map((f) => ({ + file_key: createHash('sha1').update(`deferred\n${f.name}`).digest('hex'), + name: f.name, + subject: f.subject, + poster: f.poster, + group_name: groupName, + first_article: f.first, + first_message_id: f.firstId, + last_article: f.last, + article_count: f.count, + bytes: f.bytes, + posted_at: f.postedAt, + first_seen_at: now, + last_seen_at: now, + })) + // Counts and the article range only add up within the group the file was first seen in. + await db.rawQuery( + `INSERT INTO deferred_files AS d (${Object.keys(rows[0]).join(', ')}) + SELECT * FROM jsonb_to_recordset(?::jsonb) AS r( + file_key varchar, name text, subject text, poster text, group_name text, first_article bigint, + first_message_id text, last_article bigint, article_count int, bytes bigint, posted_at timestamptz, + first_seen_at timestamptz, last_seen_at timestamptz) + ON CONFLICT (file_key) DO UPDATE SET + last_seen_at = EXCLUDED.last_seen_at, + first_message_id = CASE WHEN d.group_name = EXCLUDED.group_name AND EXCLUDED.first_article < d.first_article + THEN EXCLUDED.first_message_id ELSE d.first_message_id END, + first_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN LEAST(d.first_article, EXCLUDED.first_article) ELSE d.first_article END, + last_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN GREATEST(d.last_article, EXCLUDED.last_article) ELSE d.last_article END, + article_count = d.article_count + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.article_count ELSE 0 END, + bytes = d.bytes + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.bytes ELSE 0 END, + posted_at = LEAST(d.posted_at, EXCLUDED.posted_at)`, + [JSON.stringify(rows)] + ) + return files.size + } + + /** Takes up to `limit` files that stopped growing; they are marked `processing`. */ + async claim(limit: number): Promise { + const settled = new Date(Date.now() - indexerConfig.deferredSettleMs) + const { rows } = await db.rawQuery( + `UPDATE deferred_files SET status = 'processing', processed_at = now() + WHERE id IN ( + SELECT id FROM deferred_files + WHERE (status = 'pending' AND last_seen_at < ?) + OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}') + ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED) + RETURNING *`, + [settled, limit] + ) + return rows + } + + /** + * Queues the bodies of up to `limit` recorded ranges, oldest first: the + * articles are found again with XOVER. Returns ranges and articles queued. + */ + async expandRanges(limit: number): Promise<{ ranges: number; articles: number }> { + const ranges = await db.from('deferred_ranges').whereNull('expanded_at').orderBy('id').limit(limit) + let articles = 0 + for (const range of ranges) { + const conn = await NntpService.acquire() + let failed = false + let overviews: [number, OverviewHeader][] + try { + await conn.group(range.group_name) + overviews = (await conn.xover(String(range.first_article), String(range.last_article))).overviews + } catch (error) { + failed = true + throw error + } finally { + NntpService.release(conn, failed) + } + const jobs = overviews + .filter(([, header]) => header && isDeferred(header) && deferredName(header.subject) === null) + .map(([articleNumber, header]) => bodyJob(header, range.group_name, articleNumber)) + for (let i = 0; i < jobs.length; i += 5000) await QueueService.bodyQueue.addBulk(jobs.slice(i, i + 5000)) + await db.from('deferred_ranges').where('id', range.id).update({ expanded_at: new Date() }) + articles += jobs.length + } + return { ranges: ranges.length, articles } + } + + /** Reads one body of the file; expands small files into the body queue. Returns the new status. */ + async process(file: any): Promise { + const update: Record = { processed_at: new Date(), error: null } + try { + const conn = await NntpService.acquire() + let failed = false + let body: Buffer + try { + body = await NntpService.fetchBody(conn, file.first_message_id) + } catch (error) { + failed = true + throw error + } finally { + NntpService.release(conn, failed) + } + + const meta = parseYencMeta(body) + const totalParts = inferTotalParts(meta) + Object.assign(update, { + yenc_name: meta.begin.name, + size: meta.begin.size, + total_parts: totalParts, + sampled_part: meta.begin.part ?? 1, + }) + if ((meta.begin.part ?? 1) === 1) { + const data = decodeSegment(body, { lenientSize: true, verifyCrc: false }).data + update.content_type = sniffContentType(data) + const rar = parseRarHeaders(data) + if (rar) update.encrypted = rar.headersEncrypted || rar.entries.some((e) => e.encrypted) + } + + update.status = 'sampled' + if (totalParts <= indexerConfig.deferredExpandMaxParts && update.encrypted !== true) { + const queued = await this.expand(file) + if (queued > 0) update.status = 'expanded' + } + } catch (error: any) { + update.status = 'failed' + update.error = String(error.message).slice(0, 500) + } + await db.from('deferred_files').where('id', file.id).update(update) + return update.status as string + } + + /** Finds the file's articles again (XOVER over its range) and queues their bodies. */ + private async expand(file: any): Promise { + const first = Number(file.first_article) + const last = Number(file.last_article) + if (last - first > EXPAND_MAX_RANGE) return 0 + + const conn = await NntpService.acquire() + let failed = false + let overviews: [number, OverviewHeader][] + try { + await conn.group(file.group_name) + overviews = (await conn.xover(String(first), String(last))).overviews + } catch (error) { + failed = true + throw error + } finally { + NntpService.release(conn, failed) + } + const jobs = overviews + .filter(([, header]) => header && deferredName(header.subject) === file.name) + .map(([articleNumber, header]) => bodyJob(header, file.group_name, articleNumber)) + if (jobs.length > 0) await QueueService.bodyQueue.addBulk(jobs) + return jobs.length + } +} + +export default new DeferredService() diff --git a/app/services/FetchRangeService.ts b/app/services/FetchRangeService.ts new file mode 100644 index 0000000..e66e0a6 --- /dev/null +++ b/app/services/FetchRangeService.ts @@ -0,0 +1,115 @@ +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import QueueService from '#services/QueueService' + +/** A failed range is retried after this long... */ +const REQUEUE_AFTER_MS = 5 * 60 * 1000 +/** ...at most this many times; after that it stays visible as a gap. */ +const MAX_REQUEUES = 5 +/** A queued range without a job in Redis (e.g. Redis was flushed) is re-queued after this long. */ +const LOST_AFTER_MS = 60 * 60 * 1000 + +interface Group { + id: number + name: string +} + +/** + * Tracks the article ranges given to worker:fetch. A range only counts as + * indexed once the worker reports it done, so failed or lost fetch jobs are + * retried instead of silently skipped. See CONCEPT.md (D18). + */ +class FetchRangeService { + async queue(group: Group, startId: bigint, endId: bigint) { + const now = new Date() + const [row] = await db + .table('fetch_ranges') + .insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), created_at: now, updated_at: now }) + .returning(['id', 'requeues']) + await this.addJob(group.name, Number(row.id), 0, startId, endId) + } + + private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint) { + await QueueService.nntpFetchQueue.add( + 'fetch-headers', + { groupName, startId: startId.toString(), endId: endId.toString(), rangeId }, + // The job id changes per attempt: BullMQ ignores an add() for an id it still knows. + { jobId: this.jobId(rangeId, requeues) } + ) + } + + private jobId(rangeId: number, requeues: number) { + return `range-${rangeId}-${requeues}` + } + + async markDone(rangeId: number) { + await db.from('fetch_ranges').where('id', rangeId).update({ status: 'done', error: null, updated_at: new Date() }) + } + + async markFailed(rangeId: number, error: string) { + await db.from('fetch_ranges').where('id', rangeId).update({ status: 'failed', error, updated_at: new Date() }) + } + + /** Re-queues failed ranges, and queued ranges whose job no longer exists. */ + async requeue(group: Group): Promise { + const now = Date.now() + const candidates = await db + .from('fetch_ranges') + .where('group_id', group.id) + .where('requeues', '<', MAX_REQUEUES) + .where((q) => + q + .where((f) => f.where('status', 'failed').where('updated_at', '<', new Date(now - REQUEUE_AFTER_MS))) + .orWhere((l) => l.where('status', 'queued').where('updated_at', '<', new Date(now - LOST_AFTER_MS))) + ) + .orderBy('start_id') + + let requeued = 0 + for (const range of candidates) { + if (range.status === 'queued') { + const job = await QueueService.nntpFetchQueue.getJob(this.jobId(Number(range.id), range.requeues)) + const state = job ? await job.getState() : 'missing' + if (state !== 'missing' && state !== 'failed' && state !== 'unknown') continue + } + await db + .from('fetch_ranges') + .where('id', range.id) + .update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() }) + await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id)) + requeued++ + logger.warn( + `Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}` + ) + } + return requeued + } + + /** + * Moves `groups.last_indexed_id` up to just below the first range that is + * not done yet, and forgets done ranges. Returns the new watermark. + */ + async advanceWatermark(group: Group): Promise { + const { rows } = await db.rawQuery( + `UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce( + (SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done'), + g.last_queued_id)) + WHERE g.id = ? RETURNING last_indexed_id`, + [group.id] + ) + await db.from('fetch_ranges').where('group_id', group.id).where('status', 'done').delete() + return BigInt(rows[0].last_indexed_id) + } + + /** Ranges that failed for good (for monitoring). */ + async abandoned(group: Group) { + return db + .from('fetch_ranges') + .where('group_id', group.id) + .where('status', 'failed') + .where('requeues', '>=', MAX_REQUEUES) + .count('* as count') + .first() + } +} + +export default new FetchRangeService() diff --git a/app/services/IngestService.ts b/app/services/IngestService.ts new file mode 100644 index 0000000..d8e17aa --- /dev/null +++ b/app/services/IngestService.ts @@ -0,0 +1,471 @@ +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import type { QueryClientContract } from '@adonisjs/lucid/types/database' +import type { Knex } from 'knex' +import QueueService from '#services/QueueService' +import NameService from '#services/NameService' +import { parseXrefGroups } from '#lib/nntp/xref' +import { fileKey, parseSubject, weakFileKey } from '#lib/subject/index' +import { releaseIdentity, type ReleaseIdentity } from '#lib/release/index' +import { inferTotalParts, type YencMeta } from '#lib/yenc/index' + +/** One segment, with the identity of the file it belongs to already resolved. */ +export interface IngestSegment { + fileKey: string + poster: string + /** Subject without segment counter. */ + subject: string + name: string + totalParts: number + number: number + /** Without angle brackets. */ + messageId: string + bytes: number + articleNumber?: bigint | number + postedAt?: Date + groups: string[] + /** + * The identity comes from the yEnc name only (obfuscated posts). Such names + * can be shared by several equal-sized files of one post, so a second + * article for the same part number marks the binary as conflicted. + */ + weakIdentity?: boolean + /** Release (download) this file belongs to, when it can be determined. */ + release?: ReleaseIdentity | null + /** Exact file size from =ybegin (body-derived segments only). */ + size?: number + /** yEnc CRCs (body-derived segments only), to verify split volumes. */ + pcrc32?: string + partSize?: number + fileCrc32?: string + /** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */ + md5_16k?: string +} + +export interface CompletedBinary { + id: number + fileKey: string +} + +const BINARY_CHUNK = 1000 +/** last_seen_at is only rewritten when it is older than this (it drives GC at 48 h). */ +const TOUCH_MS = 10 * 60 * 1000 + +type Merge = 'least' | 'coalesce' | 'touch' + +/** + * Inserts rows keyed by `key` and returns their ids. Existing rows are not + * locked unless something in them changes (`least`: an earlier value, + * `coalesce`: a value where there was none, `touch`: a timestamp more than + * TOUCH_MS newer). Parallel fetch jobs see mostly the same rows, and + * `INSERT … ON CONFLICT DO UPDATE` made them wait on each other for every one. + */ +async function upsertKeyed( + knex: Knex, + table: 'binaries' | 'releases', + key: string, + rows: Record[], + merge: Record +): Promise> { + const ids = new Map() + if (rows.length === 0) return ids + const inserted: Record[] = await knex(table).insert(rows).onConflict(key).ignore().returning(['id', key]) + for (const row of inserted) ids.set(row[key], Number(row.id)) + + const incoming = new Map(rows.filter((r) => !ids.has(r[key] as string)).map((r) => [r[key] as string, r])) + if (incoming.size === 0) return ids + const columns = Object.keys(merge) + const existing: Record[] = await knex(table).whereIn(key, [...incoming.keys()]).select('id', key, ...columns) + + const changes: Record[] = [] + for (const row of existing) { + ids.set(row[key], Number(row.id)) + const next = incoming.get(row[key])! + const changed = columns.some((column) => { + const [kind] = merge[column] + const current = row[column] + const value = next[column] + if (value === null || value === undefined) return false + if (kind === 'coalesce') return current === null + if (kind === 'least') return current === null || value < current + return current === null || +(value as Date) - +current > TOUCH_MS + }) + if (changed) changes.push({ id: Number(row.id), ...Object.fromEntries(columns.map((c) => [c, next[c] ?? null])) }) + } + if (changes.length === 0) return ids + + const set = columns + .map((c) => { + const [kind] = merge[c] + const fn = kind === 'least' ? 'least' : kind === 'touch' ? 'greatest' : 'coalesce' + return `${c} = ${fn}(t.${c}, v.${c})` + }) + .join(', ') + const types = columns.map((c) => `${c} ${merge[c][1]}`).join(', ') + changes.sort((a, b) => (a.id as number) - (b.id as number)) + for (let attempt = 1; ; attempt++) { + try { + await knex.raw( + `UPDATE ${table} t SET ${set} + FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, ${types}) + WHERE t.id = v.id`, + [JSON.stringify(changes)] + ) + return ids + } catch (error: any) { + // Rare now, but two jobs can still update overlapping rows in a different order. + if (error.code !== '40P01' || attempt >= 3) throw error + } + } +} +/** Size-only matching below this is too ambiguous (nfo, sfv, small volumes). */ +export const PAR2_SIZE_ONLY_MIN = 1024 * 1024 +/** A par2 set and its files are posted together; don't match across days. */ +export const PAR2_MATCH_WINDOW_SECONDS = 12 * 60 * 60 +const PART_CHUNK = 2000 + +function chunks(items: T[], size: number): T[][] { + const result: T[][] = [] + for (let i = 0; i < items.length; i += size) result.push(items.slice(i, i + size)) + return result +} + +function validDate(date?: Date) { + return date && !Number.isNaN(date.getTime()) ? date : null +} + +/** An XOVER overview entry as returned by nntp-js. */ +export type OverviewHeader = Record + +function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) { + const xrefGroups = parseXrefGroups(header.xref) + return { + poster: header.from ?? '', + messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''), + bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0, + articleNumber, + postedAt: header.date ? new Date(header.date) : undefined, + groups: xrefGroups.length > 0 ? xrefGroups : [groupName], + } +} + +/** Builds a segment from the subject line; null when the subject is obfuscated. */ +export function segmentFromHeader( + header: OverviewHeader, + groupName: string, + articleNumber?: bigint | number +): IngestSegment | null { + const parsed = header.subject ? parseSubject(header.subject) : null + if (!parsed || !header['message-id']) return null + const base = common(header, groupName, articleNumber) + return { + ...base, + fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments), + subject: parsed.base, + name: parsed.name, + totalParts: parsed.totalSegments, + number: parsed.segment, + release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }), + } +} + +/** + * Builds a segment for an obfuscated post from the yEnc header in its body. + * Subject and poster can be random per article, so the identity comes from + * the yEnc header only (see weakFileKey). + */ +export function segmentFromYenc( + header: OverviewHeader, + groupName: string, + meta: Pick & Partial>, + articleNumber?: bigint | number +): IngestSegment | null { + const base = common(header, groupName, articleNumber) + const totalParts = inferTotalParts(meta) + const number = meta.begin.part ?? 1 + if (!base.messageId || number < 1 || number > totalParts) return null + const subject = `"${meta.begin.name}" yEnc` + return { + ...base, + fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts), + subject, + name: meta.begin.name, + totalParts, + number, + weakIdentity: true, + release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }), + size: meta.begin.size, + // Single-part posts only carry crc32, which is then also the part's CRC. + pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32), + partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size, + fileCrc32: meta.end?.crc32, + } +} + +/** + * Stores segments in `binaries`/`parts` and queues every binary that became + * complete exactly once. Idempotent: ingesting the same segments again is a + * no-op. See CONCEPT.md (D12). + */ +class IngestService { + async ingest(segments: IngestSegment[]): Promise { + if (segments.length === 0) return [] + + // Short statements, no enclosing transaction: every step is idempotent, and + // row locks held for a whole batch made parallel fetch jobs wait on each other. + const binaryIds = await (async () => { + const knex = db.connection().getWriteClient() + const now = new Date() + const releaseIds = await this.upsertReleases(knex, segments, now) + + // One row per file_key (ON CONFLICT cannot touch a row twice per statement), + // sorted so concurrent workers lock rows in the same order and cannot deadlock. + const binaries = new Map>() + for (const segment of segments) { + const postedAt = validDate(segment.postedAt) + const existing = binaries.get(segment.fileKey) + if (existing) { + if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) { + existing.posted_at = postedAt + } + continue + } + binaries.set(segment.fileKey, { + file_key: segment.fileKey, + poster: segment.poster, + subject: segment.subject, + name: segment.name, + total_parts: segment.totalParts, + groups: JSON.stringify(segment.groups), + posted_at: postedAt, + first_seen_at: now, + last_seen_at: now, + release_id: segment.release ? releaseIds.get(segment.release.key) : null, + size: segment.size ?? null, + md5_16k: segment.md5_16k ?? null, + }) + } + const binaryRows = [...binaries.values()].sort((a, b) => + (a.file_key as string).localeCompare(b.file_key as string) + ) + + const idsByKey = new Map() + for (const chunk of chunks(binaryRows, BINARY_CHUNK)) { + const ids = await upsertKeyed(knex, 'binaries', 'file_key', chunk, { + last_seen_at: ['touch', 'timestamptz'], + posted_at: ['least', 'timestamptz'], + release_id: ['coalesce', 'bigint'], + md5_16k: ['coalesce', 'varchar'], + }) + for (const [key, id] of ids) idsByKey.set(key, id) + } + + const partRows = segments + .map((segment) => ({ + binary_id: idsByKey.get(segment.fileKey)!, + number: segment.number, + message_id: segment.messageId, + bytes: segment.bytes, + article_number: segment.articleNumber?.toString() ?? null, + pcrc32: segment.pcrc32 ?? null, + part_size: segment.partSize ?? null, + file_crc32: segment.fileCrc32 ?? null, + md5_16k: segment.md5_16k ?? null, + })) + const weakRows = partRows.filter((_, i) => segments[i].weakIdentity) + + partRows.sort((a, b) => a.binary_id - b.binary_id || a.number - b.number) + for (const chunk of chunks(partRows, PART_CHUNK)) { + await knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore() + } + + if (weakRows.length > 0) await this.flagConflicts(db.connection(), weakRows) + + return [...idsByKey.values()] + })() + + // par2 naming needs a size, which only body-derived segments have; for + // header batches these queries were most of Postgres' work. + const sized = [...new Set(segments.filter((s) => s.size != null).map((s) => s.fileKey))] + if (sized.length > 0) { + const sizedIds = (await db.from('binaries').whereIn('file_key', sized).select('id')).map((r) => Number(r.id)) + await this.matchPar2Releases(sizedIds) + await NameService.resolve(sizedIds) + } + + // After commit, so the last writer of a binary always sees all of its parts. + const completed = await this.claimCompleted(binaryIds) + await this.enqueue(completed) + return completed + } + + /** + * Creates or touches the releases referenced by `segments` and returns + * their ids by release key. Also used by `release:rebuild`. + */ + async upsertReleases( + knex: Knex, + segments: Pick[], + now = new Date() + ): Promise> { + const releases = new Map>() + for (const segment of segments) { + const release = segment.release + if (!release) continue + const postedAt = validDate(segment.postedAt) + const existing = releases.get(release.key) + if (existing) { + if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) { + existing.posted_at = postedAt + } + // Not every file carries the [i/M] counter (e.g. a par2 index): take it from any that does. + if (existing.file_total === null && release.fileTotal) existing.file_total = release.fileTotal + continue + } + releases.set(release.key, { + release_key: release.key, + name: release.name, + // Obfuscating posters randomise the poster, so it says nothing about the release. + poster: segment.weakIdentity ? null : segment.poster, + file_total: release.fileTotal ?? null, + source: release.source, + groups: JSON.stringify(segment.groups), + posted_at: postedAt, + first_seen_at: now, + last_seen_at: now, + }) + } + + const ids = new Map() + const rows = [...releases.values()].sort((a, b) => + (a.release_key as string).localeCompare(b.release_key as string) + ) + for (const chunk of chunks(rows, BINARY_CHUNK)) { + const result = await upsertKeyed(knex, 'releases', 'release_key', chunk, { + last_seen_at: ['touch', 'timestamptz'], + posted_at: ['least', 'timestamptz'], + file_total: ['coalesce', 'integer'], + source: ['coalesce', 'varchar'], + }) + for (const [key, id] of result) ids.set(key, id) + } + return ids + } + + /** + * Assigns body-derived binaries without a release and without a + * fingerprint to a known par2 set by size: only when that size is listed + * once in the set, the file is large, and exactly one set nearby lists it. + * Binaries with a fingerprint are matched exactly by NameService instead. + */ + async matchPar2Releases(binaryIds: number[]) { + if (binaryIds.length === 0) return + await db.rawQuery( + `UPDATE binaries b SET release_id = m.release_id + FROM ( + SELECT b2.id, min(rf.release_id) AS release_id + FROM binaries b2 + JOIN release_files rf ON rf.size = b2.size + JOIN releases r ON r.id = rf.release_id + WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ? + AND abs(extract(epoch FROM (r.posted_at - b2.posted_at))) < ? + AND (SELECT count(*) FROM release_files x WHERE x.release_id = rf.release_id AND x.size = rf.size) = 1 + GROUP BY b2.id + HAVING count(DISTINCT rf.release_id) = 1 + ) m + WHERE b.id = m.id`, + [binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS] + ) + } + + /** + * Flags binaries where a stored part has a different message-id and a + * different part CRC than the incoming one: two files were merged under + * one identity. Identical reposts (same CRC) are ignored. + */ + private async flagConflicts( + trx: QueryClientContract, + rows: { + binary_id: number + number: number + message_id: string + bytes: number + article_number: string | null + pcrc32: string | null + part_size: number | null + file_crc32: string | null + md5_16k: string | null + }[] + ) { + const bindings = [ + rows.map((r) => r.binary_id), + rows.map((r) => r.number), + rows.map((r) => r.message_id), + rows.map((r) => r.bytes), + rows.map((r) => r.article_number), + rows.map((r) => r.pcrc32), + rows.map((r) => r.part_size), + rows.map((r) => r.file_crc32), + rows.map((r) => r.md5_16k), + ] + // Keep the extra articles: SplitService separates the volumes from them. + const conflicts = await trx.rawQuery( + `INSERT INTO part_conflicts (binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k) + SELECT v.binary_id, v.number, v.message_id, v.bytes, v.article_number, v.pcrc32, v.part_size, v.file_crc32, v.md5_16k + FROM unnest(?::bigint[], ?::int[], ?::text[], ?::int[], ?::bigint[], ?::text[], ?::int[], ?::text[], ?::text[]) + AS v(binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k) + JOIN parts p ON p.binary_id = v.binary_id AND p.number = v.number + WHERE p.message_id <> v.message_id + -- Same part CRC: the same data posted again (a repost), not another file. + AND (p.pcrc32 IS NULL OR v.pcrc32 IS NULL OR p.pcrc32 <> v.pcrc32) + ON CONFLICT DO NOTHING + RETURNING binary_id`, + bindings as any + ) + const binaryIds: number[] = [ + ...new Set(conflicts.rows.map((r: { binary_id: string }) => Number(r.binary_id))), + ] + if (binaryIds.length === 0) return + const result = await trx.rawQuery( + `UPDATE binaries SET conflicted_at = now() + WHERE conflicted_at IS NULL AND id = ANY(?) RETURNING name`, + [binaryIds] + ) + for (const row of result.rows) { + logger.warn(`Binary "${row.name}" has conflicting parts; it will be split into volumes.`) + } + } + + /** + * Marks complete binaries among `binaryIds` as claimed. The single UPDATE + * locks each row, so concurrent callers never claim the same binary twice. + */ + async claimCompleted(binaryIds: number[]): Promise { + if (binaryIds.length === 0) return [] + const result = await db.rawQuery( + `UPDATE binaries b SET completed_at = now() + WHERE b.id = ANY(?) AND b.completed_at IS NULL AND b.conflicted_at IS NULL + AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) >= b.total_parts + RETURNING b.id, b.file_key`, + [binaryIds] + ) + return result.rows.map((row: { id: string; file_key: string }) => ({ + id: Number(row.id), + fileKey: row.file_key, + })) + } + + async enqueue(completed: CompletedBinary[]) { + if (completed.length === 0) return + // jobId = file_key: BullMQ ignores an add() for an id that already exists. + await QueueService.fileQueue.addBulk( + completed.map(({ id, fileKey }) => ({ + name: 'process-file', + data: { binaryId: id }, + opts: { jobId: fileKey }, + })) + ) + } +} + +export default new IngestService() diff --git a/app/services/NameService.ts b/app/services/NameService.ts new file mode 100644 index 0000000..ae4f1a0 --- /dev/null +++ b/app/services/NameService.ts @@ -0,0 +1,128 @@ +import db from '@adonisjs/lucid/services/db' +import { looksRandom } from '#services/ArchiveService' + +/** + * A release that is nothing more than one randomly named file (its own + * filename stem). Such a file moves into the par2 set that names it exactly; + * real groupings (par2, RAR, NZB, spot, subject, post id) are left alone. + */ +export const WEAK_RELEASE_SQL = (column: string) => + `EXISTS (SELECT 1 FROM releases wr WHERE wr.id = ${column} AND wr.source IN ('stem', 'weak-stem') + AND wr.file_count <= 1 AND (wr.name !~ '[._ -]' OR NOT is_readable_name(wr.name)))` + +/** + * Whether `candidate` (e.g. from a par2 listing) should replace `current`: + * the current name is random, or a single token without separators + * (mOzyyxCyYUj, F8lv3w5agMDI3; too short for looksRandom), while the + * candidate reads like a release name. + */ +export function isBetterName(current: string, candidate: string): boolean { + if (looksRandom(candidate) || !/[._ -]/.test(candidate)) return false + return looksRandom(current) || !/[._ -]/.test(current) +} + +/** Smaller files (nfo, sfv) collide too easily on size alone. */ +const MIN_SIZE_ONLY = 64 * 1024 + +/** + * Gives binaries (and their files) the real filename that a par2 set lists + * for them. See CONCEPT.md (D21). + * - Exact: size and MD5 of the first 16 KiB both match a par2 entry. This + * also puts the binary in that par2 set's release. + * - Size only, when the fingerprint is unknown: within the binary's own + * release, only when exactly one listed file has that size. + */ +class NameService { + async resolve(binaryIds: number[]): Promise { + if (binaryIds.length === 0) return 0 + const exact = await db.rawQuery( + `UPDATE binaries b SET real_name = m.name, + release_id = CASE WHEN b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')} THEN m.release_id ELSE b.release_id END + FROM ( + SELECT DISTINCT ON (b2.id) b2.id, rf.name, rf.release_id + FROM binaries b2 + JOIN release_files rf ON rf.size = b2.size AND rf.md5_16k = b2.md5_16k + WHERE b2.id = ANY(?) + ORDER BY b2.id, rf.release_id DESC + ) m + WHERE b.id = m.id AND (b.real_name IS DISTINCT FROM m.name OR b.release_id IS NULL + OR (b.release_id <> m.release_id AND ${WEAK_RELEASE_SQL('b.release_id')}))`, + [binaryIds] + ) + const bySize = await db.rawQuery( + `UPDATE binaries b SET real_name = m.name + FROM ( + SELECT b2.id, min(rf.name) AS name + FROM binaries b2 + JOIN release_files rf ON rf.release_id = b2.release_id AND rf.size = b2.size + WHERE b2.id = ANY(?) AND b2.real_name IS NULL AND b2.md5_16k IS NULL AND b2.size >= ? + GROUP BY b2.id + HAVING count(DISTINCT rf.name) = 1 + ) m + WHERE b.id = m.id`, + [binaryIds, MIN_SIZE_ONLY] + ) + await db.rawQuery( + `UPDATE files f SET real_name = b.real_name, + release_id = CASE WHEN f.release_id IS NULL OR ${WEAK_RELEASE_SQL('f.release_id')} THEN coalesce(b.release_id, f.release_id) ELSE f.release_id END + FROM binaries b + WHERE b.id = ANY(?) AND b.file_id = f.id + AND (f.real_name IS DISTINCT FROM b.real_name OR (f.release_id IS NULL AND b.release_id IS NOT NULL) + OR (f.release_id <> b.release_id AND ${WEAK_RELEASE_SQL('f.release_id')}))`, + [binaryIds] + ) + return (exact.rowCount ?? 0) + (bySize.rowCount ?? 0) + } + + /** + * When a par2 set arrives: the binaries it can name, i.e. those matching + * one of its entries exactly (size + first-16-KiB MD5) and those in its + * release. Selecting every binary of the listed sizes instead picked up + * hundreds of thousands of equal-sized volumes per set (LOG.md 2026-09-28 (18)). + */ + async resolveForRelease(releaseId: number): Promise { + const { rows } = await db.rawQuery( + `SELECT b.id FROM release_files rf + JOIN binaries b ON b.md5_16k = rf.md5_16k AND b.size = rf.size + WHERE rf.release_id = ? + UNION + SELECT id FROM binaries WHERE release_id = ? AND size IS NOT NULL`, + [releaseId, releaseId] + ) + return this.resolve(rows.map((r: { id: string }) => Number(r.id))) + } +} + +/** + * Repair for data from before files moved into the par2 set that names them + * (used by `release:rename`): regroups exact matches out of random-named + * single-file releases, then renames par2 releases that still carry a + * random name after their listing. No downloads. + */ +export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; renamed: number }> { + const { rows } = await db.rawQuery( + `SELECT DISTINCT b.id FROM binaries b + JOIN release_files rf ON rf.md5_16k = b.md5_16k AND rf.size = b.size + WHERE b.release_id IS DISTINCT FROM rf.release_id + AND (b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')})` + ) + const ids = rows.map((r: { id: string }) => Number(r.id)) + for (let i = 0; i < ids.length; i += 1000) await nameService.resolve(ids.slice(i, i + 1000)) + + const { par2ReleaseName } = await import('#lib/release/index') + const listings = await db.rawQuery( + `SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r + JOIN release_files rf ON rf.release_id = r.id GROUP BY r.id, r.name` + ) + let renamed = 0 + for (const release of listings.rows) { + const better = par2ReleaseName(release.names) + if (!better || better === release.name || !isBetterName(release.name, better)) continue + await db.from('releases').where('id', release.id).update({ name: better }) + renamed++ + } + return { regrouped: ids.length, renamed } +} + +const nameService = new NameService() +export default nameService diff --git a/app/services/NntpService.ts b/app/services/NntpService.ts index 1daa2b4..2981d88 100644 --- a/app/services/NntpService.ts +++ b/app/services/NntpService.ts @@ -1,5 +1,6 @@ import nntpConfig from '#config/nntp' import { NNTP } from "nntp-js"; +import { parseGroupResponse, type GroupInfo } from '#lib/nntp/group' class NntpService { private readonly poolSize: number; @@ -14,15 +15,20 @@ class NntpService { this.idleConnections = []; this.waiters = []; this.createdCount = 0; - console.log(`NNTP Pool initialized with size ${this.poolSize}`) } private async _createConnection() { - // This connection logic is based on the older, working pool implementation. const conn = new NNTP(nntpConfig.host, nntpConfig.port); - await conn.connect(); - if (nntpConfig.user) { - await conn.login(nntpConfig.user, nntpConfig.password?.release()); + try { + await conn.connect(); + if (nntpConfig.user) { + await conn.login(nntpConfig.user, nntpConfig.password?.release()); + } + } catch (error) { + // Free the slot, otherwise a failed connect shrinks the pool for good. + this.createdCount--; + this._destroy(conn); + throw error; } this.allConnections.add(conn); return conn; @@ -30,38 +36,79 @@ class NntpService { public async acquire() { if (this.idleConnections.length > 0) { - console.log('Reusing existing connection from pool.'); return this.idleConnections.pop(); } if (this.createdCount < this.poolSize) { this.createdCount++; - console.log(`Creating new connection (${this.createdCount}/${this.poolSize}).`); + console.log(`Creating new connection (${this.createdCount}/${this.poolSize}).`); return this._createConnection(); } - console.log(`Pool maxed out at ${this.poolSize}. Waiting for a connection to become available.`); return new Promise(resolve => this.waiters.push(resolve)); } - public release(conn: any) { - if (this.waiters.length > 0) { - console.log('Releasing connection directly to a waiting task.'); - const resolve = this.waiters.shift(); - if(resolve) resolve(conn); + /** + * Returns a connection to the pool. Pass `failed` when a command on it + * threw: the connection may be dead (e.g. closed by the server while idle), + * so it is discarded instead of being handed to the next job. + */ + public release(conn: any, failed = false) { + if (failed) { + this.discard(conn); + return; + } + const resolve = this.waiters.shift(); + if (resolve) { + resolve(conn); } else { - console.log('Returning connection to the idle pool.'); this.idleConnections.push(conn); } } - public async shutdown() { - console.log('Shutting down all connections in the pool.'); - const shutdownPromises: Promise[] = []; - for (const conn of this.allConnections) { - shutdownPromises.push(conn.quit()); + /** Closes a broken connection and frees its slot for a new one. */ + public discard(conn: any) { + this.allConnections.delete(conn); + this.createdCount--; + this._destroy(conn); + + // Someone is waiting for a slot: open a fresh connection for them. + const resolve = this.waiters.shift(); + if (resolve) { + this.createdCount++; + this._createConnection().then(resolve, () => this.waiters.unshift(resolve)); } - await Promise.all(shutdownPromises); + } + + private _destroy(conn: any) { + try { + conn.sock?.destroy(); + } catch { + // already closed + } + } + + /** Selects a group and returns its correctly parsed article range. */ + public async selectGroup(conn: any, name: string): Promise { + const { resp } = await conn.group(name) + return parseGroupResponse(resp) + } + + /** Fetches a raw (still dot-stuffed) article body as a Buffer. */ + public async fetchBody(conn: any, messageId: string): Promise { + const id = messageId.startsWith('<') ? messageId : `<${messageId}>` + const { data } = await conn.body(id) + return Buffer.from(data.buffer, data.byteOffset, data.byteLength) + } + + public async shutdown() { + console.log('Shutting down all connections in the pool.'); + // A connection the server already closed may never answer QUIT. + const quit = (conn: any) => + Promise.race([conn.quit(), new Promise((resolve) => setTimeout(resolve, 2000).unref())]) + .catch(() => {}) + .finally(() => this._destroy(conn)); + await Promise.all([...this.allConnections].map(quit)); this.allConnections.clear(); this.idleConnections.length = 0; this.waiters.length = 0; @@ -69,4 +116,4 @@ class NntpService { } } -export default new NntpService(); +export default new NntpService(nntpConfig.poolSize); diff --git a/app/services/NzbImportService.ts b/app/services/NzbImportService.ts new file mode 100644 index 0000000..bcdba0e --- /dev/null +++ b/app/services/NzbImportService.ts @@ -0,0 +1,168 @@ +import { createHash } from 'node:crypto' +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import IngestService from '#services/IngestService' +import { type ReleaseIdentity } from '#lib/release/index' +import { parseNzb, type ParsedNzb } from '#lib/nzb/index' +import { parseSubject } from '#lib/subject/index' +import { fileStem } from '#lib/release/index' +import { looksRandom } from '#services/ArchiveService' + +export interface NzbImportResult { + releaseId: number + releaseName: string + listed: number + known: number + added: number + password: boolean +} + +/** Filename of an NZB entry, from its subject (tolerating odd quoting). */ +export function nzbFileName(subject: string): string { + const parsed = parseSubject(subject) + const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject + return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim() +} + +export interface NzbImportOptions { + /** + * For NZBs inside an archive: the entry's name. It keeps several NZBs of + * one archive apart and names their release when the NZB has no name. + */ + entryName?: string + /** Move the posted file into the release the NZB describes (default true). */ + moveContainer?: boolean + /** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */ + name?: string + /** Release source for new releases (default `nzb`). */ + source?: ReleaseIdentity['source'] +} + +/** + * Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26). + * Files it lists that are already known (by first segment) keep their + * release, which the NZB then joins; the others are added as files, so posts + * spread over groups we do not index still become complete. + */ +class NzbImportService { + async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise { + const nzb = parseNzb(xml) + if (!nzb || nzb.files.length === 0) return null + + const entries = nzb.files.map((f) => ({ + file: f, + name: nzbFileName(f.subject), + firstId: f.segments[0].id, + })) + + return db.transaction(async (trx) => { + const known = await trx + .from('files') + .whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)]) + .select('id', 'release_id', `message_ids`) + const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id)) + + // Join the release the listed files are already in, else the posted NZB's own, else a new one. + const counts = new Map() + for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1) + let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] + const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName) + if (!releaseId) { + const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}` + const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex') + const ids = await IngestService.upsertReleases(trx.knexClient, [ + { + release: { key, name: title, source: options.source ?? 'nzb' }, + poster: nzbFile.poster, + groups: nzbFile.groups ?? [], + postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined, + }, + ]) + releaseId = ids.get(key)! + } + + const now = new Date() + const missing = entries.filter((e) => !knownIds.has(e.firstId)) + for (let i = 0; i < missing.length; i += 500) { + await trx + .knexClient('files') + .insert( + missing.slice(i, i + 500).map(({ file, name, firstId }) => ({ + file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'), + filename: name, + poster: file.poster || nzbFile.poster, + date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000, + parts: file.segments.length, + size: file.segments.reduce((sum, s) => sum + s.bytes, 0), + message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))), + groups: JSON.stringify(file.groups), + release_id: releaseId, + created_at: now, + updated_at: now, + })) + ) + .onConflict('file_key') + .ignore() + } + // The poster's NZB says which files form the post: known files join this + // release, also when name rules had split them over several. + const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))] + await trx + .from('files') + .whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id)) + .update({ release_id: releaseId }) + if (previous.length) { + await trx.rawQuery( + `UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`, + [releaseId, releaseId, releaseId] + ) + await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete() + } + // The NZB file joins the release it describes, unless it is part of a real release already. + const own = nzbFile.release_id ? Number(nzbFile.release_id) : null + if (own !== releaseId && options.moveContainer !== false) { + const ownRelease = own ? await trx.from('releases').where('id', own).first() : null + if (!ownRelease || ownRelease.file_count <= 1) { + await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId }) + if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete() + } + } + + const release = await trx.from('releases').where('id', releaseId).first() + const update: Record = {} + if (nzb.meta.password) update.password = nzb.meta.password + if (!release.file_total) update.file_total = entries.length + if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title + if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update) + + const result = { + releaseId, + releaseName: (update.name as string) ?? release.name, + listed: entries.length, + known: known.length, + added: missing.length, + password: Boolean(nzb.meta.password), + } + logger.info( + `NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` + + `${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.` + ) + return result + }) + } + + /** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */ + private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string { + if (nzb.meta.name) return nzb.meta.name + const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : '' + if (entry && !looksRandom(entry)) return entry + const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? '')) + if (own && !looksRandom(own)) return own + const largest = nzb.files + .map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) })) + .sort((a, b) => b.bytes - a.bytes)[0] + return fileStem(largest?.name ?? own) + } +} + +export default new NzbImportService() diff --git a/app/services/Par2Service.ts b/app/services/Par2Service.ts new file mode 100644 index 0000000..957f072 --- /dev/null +++ b/app/services/Par2Service.ts @@ -0,0 +1,165 @@ +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import NntpService from '#services/NntpService' +import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService' +import { decodeSegment, YencFile } from '#lib/yenc/index' +import { isPar2, parsePar2, type Par2Set } from '#lib/par2/index' +import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index' +import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService' + +/** Par2 index files are small; recovery volumes can be large and are not needed. */ +const MAX_PARTS = 20 +/** Size-only matching below this is too ambiguous. */ +const SIZE_ONLY_MIN = 1024 * 1024 + +function countBy(values: T[]): Map { + const counts = new Map() + for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1) + return counts +} + +export interface Par2Result { + releaseId: number + releaseName: string + listed: number + matched: number +} + +/** + * Recognises releases of obfuscated posts from their par2 files: the par2 + * lists the real name and exact size of every file in the set, and + * body-derived binaries carry their exact size. See CONCEPT.md (D16). + */ +class Par2Service { + /** + * Reads a par2 file whose first segment is already decoded, downloading the + * remaining parts if there are any. Returns null when it is not par2. + */ + async read(first: Buffer, messageIds: Record): Promise { + if (!isPar2(first)) return null + const ids = Object.entries(messageIds) + .sort(([a], [b]) => Number(a) - Number(b)) + .map(([, part]) => part.id) + if (ids.length > MAX_PARTS) return null + if (ids.length === 1) return parsePar2(first) + + let assembled: YencFile | undefined + const conn = await NntpService.acquire() + let failed = false + try { + for (const id of ids) { + const segment = decodeSegment(await NntpService.fetchBody(conn, id)) + assembled ??= YencFile.fromSegment(segment) + assembled.add(segment) + } + } catch (error) { + failed = true + throw error + } finally { + NntpService.release(conn, failed) + } + return parsePar2(assembled!.getBuffer()) + } + + async attach(binary: any, set: Par2Set): Promise { + const result = await db.transaction(async (trx) => { + // Exact: size and first-16-KiB MD5 of a listed file. Only these may pull + // the set into a release the files already belong to; size alone is + // ambiguous across posts (standard volume sizes recur). + const exact = await trx + .from('binaries') + .whereIn(['size', 'md5_16k'] as any, set.files.map((f) => [f.length, f.md5_16k]) as any) + .select('id', 'release_id') + + // Size only, for files without a fingerprint: the size must be listed + // once in this set, be large, and belong to exactly one binary nearby. + const listedOnce = [...countBy(set.files.map((f) => f.length))] + .filter(([size, n]) => n === 1 && size >= SIZE_ONLY_MIN) + .map(([size]) => size) + const bySize = listedOnce.length + ? await trx + .from('binaries') + .whereIn('size', listedOnce) + .whereNull('md5_16k') + .whereRaw('abs(extract(epoch FROM (posted_at - ?::timestamptz))) < ?', [ + binary.posted_at, + PAR2_MATCH_WINDOW_SECONDS, + ]) + .select('id', 'release_id', 'size') + : [] + const sizeCounts = countBy(bySize.map((b) => Number(b.size))) + const fuzzy = bySize.filter((b) => sizeCounts.get(Number(b.size)) === 1) + const candidates = [...exact, ...fuzzy] + + // Prefer a release the exactly matched files are already in (e.g. + // obfuscated volumes grouped by their stem); otherwise the set's own. + const counts = countBy(exact.filter((c) => c.release_id).map((c) => Number(c.release_id))) + let releaseId = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0] + const identity = par2ReleaseIdentity( + set.recoverySetId, + set.files.map((f) => f.name) + ) + if (!releaseId) { + const ids = await IngestService.upsertReleases(trx.knexClient, [ + { + release: identity, + poster: binary.poster, + weakIdentity: true, + groups: binary.groups, + postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined, + }, + ]) + releaseId = ids.get(identity.key)! + } + + await trx + .knexClient('release_files') + .insert( + set.files.map((f) => ({ + release_id: releaseId, + par2_file_id: f.fileId, + name: f.name, + size: f.length, + md5_16k: f.md5_16k, + md5: f.md5, + set_id: set.recoverySetId, + })) + ) + .onConflict(['release_id', 'par2_file_id']) + .ignore() + + const matchIds = candidates.filter((c) => !c.release_id).map((c) => Number(c.id)) + await trx.from('binaries').whereIn('id', [binary.id, ...matchIds]).whereNull('release_id').update({ release_id: releaseId }) + // Exact matches sitting alone in a random-named release (one file of a + // season, each posted under its own random name) join the set's release. + await trx.rawQuery( + `UPDATE binaries b SET release_id = ? WHERE b.id = ANY(?) AND b.release_id <> ? AND ${WEAK_RELEASE_SQL('b.release_id')}`, + [releaseId, exact.map((c) => Number(c.id)), releaseId] + ) + await trx.rawQuery( + `UPDATE files f SET release_id = b.release_id FROM binaries b + WHERE b.file_id = f.id AND b.release_id = ? AND f.release_id IS DISTINCT FROM b.release_id`, + [releaseId] + ) + + // The set may have joined a release named after one random filename. + let releaseName = (await trx.from('releases').where('id', releaseId).first())?.name + const better = par2ReleaseName(set.files.map((f) => f.name)) + if (releaseName && better && isBetterName(releaseName, better)) { + await trx.from('releases').where('id', releaseId).update({ name: better }) + releaseName = better + } + logger.info( + `Par2 set ${set.recoverySetId.slice(0, 8)} → release ${releaseId} "${releaseName}": ` + + `${set.files.length} files listed, ${matchIds.length} binaries matched by size.` + ) + return { releaseId, releaseName, listed: set.files.length, matched: matchIds.length } + }) + + // Real names for every known file of the set (exact fingerprint, or size within the release). + await NameService.resolveForRelease(result.releaseId) + return result + } +} + +export default new Par2Service() diff --git a/app/services/QueueService.ts b/app/services/QueueService.ts index 5824c11..2adf00c 100644 --- a/app/services/QueueService.ts +++ b/app/services/QueueService.ts @@ -1,29 +1,60 @@ -import { Queue } from 'bullmq' +import { Queue, type DefaultJobOptions } from 'bullmq' import queueConfig from '#config/queue' +import gcConfig from '#config/gc' +const defaultJobOptions: DefaultJobOptions = { + // Transient NNTP errors (e.g. a pooled connection the server closed) are retried. + attempts: 3, + backoff: { type: 'exponential', delay: 2000 }, + removeOnComplete: { + age: gcConfig.jobs.completedAgeSeconds, + count: gcConfig.jobs.completedMaxCount, + }, + removeOnFail: { age: gcConfig.jobs.failedAgeSeconds }, +} + +/** + * Queues are created (and connect to Redis) on first use. Ace imports every + * command at boot, so eager queues would keep one-off commands from exiting. + */ class QueueService { - public readonly nntpFetchQueue: Queue - public readonly headerQueue: Queue - public readonly fileQueue: Queue - public readonly bodyQueue: Queue - public readonly collectionQueue: Queue + private readonly queues = new Map() - constructor() { - this.nntpFetchQueue = new Queue('nntp-fetch-queue', { connection: queueConfig.connection }) - this.headerQueue = new Queue('header-queue', { connection: queueConfig.connection }) - this.fileQueue = new Queue('file-queue', { connection: queueConfig.connection }) - this.bodyQueue = new Queue('body-queue', { connection: queueConfig.connection }) - this.collectionQueue = new Queue('collection-queue', { connection: queueConfig.connection }) + private queue(name: string): Queue { + let queue = this.queues.get(name) + if (!queue) { + queue = new Queue(name, { connection: queueConfig.connection, defaultJobOptions }) + this.queues.set(name, queue) + } + return queue } + get nntpFetchQueue() { + return this.queue('nntp-fetch-queue') + } + get fileQueue() { + return this.queue('file-queue') + } + get bodyQueue() { + return this.queue('body-queue') + } + get collectionQueue() { + return this.queue('collection-queue') + } + + /** Queues no longer used by the pipeline, kept here so GC can empty them. */ + get legacy(): Queue[] { + return [this.queue('header-queue')] + } + + get all(): Queue[] { + return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue] + } + + /** Closes the queues that were opened. */ async closeAll() { - await Promise.all([ - this.nntpFetchQueue.close(), - this.headerQueue.close(), - this.fileQueue.close(), - this.bodyQueue.close(), - this.collectionQueue.close(), - ]) + await Promise.all([...this.queues.values()].map((queue) => queue.close())) + this.queues.clear() } } diff --git a/app/services/RedisService.ts b/app/services/RedisService.ts index fc75fcd..656b211 100644 --- a/app/services/RedisService.ts +++ b/app/services/RedisService.ts @@ -4,11 +4,21 @@ import redisConfig from '#config/redis' const require = createRequire(import.meta.url) const IORedis = require('ioredis') +/** + * Connects on first use. Ace imports every command at boot, so an eager + * connection would keep even `node ace list` from exiting. + */ class RedisService { - public readonly client: any + private instance?: any - constructor() { - this.client = new IORedis(redisConfig.connections.main) + public get client(): any { + this.instance ??= new IORedis(redisConfig.connections.main) + return this.instance + } + + async close() { + if (this.instance) await this.instance.quit() + this.instance = undefined } } diff --git a/app/services/SearchService.ts b/app/services/SearchService.ts new file mode 100644 index 0000000..6d0ad29 --- /dev/null +++ b/app/services/SearchService.ts @@ -0,0 +1,284 @@ +import db from '@adonisjs/lucid/services/db' +import { buildNzb } from '#lib/nzb/index' +import SpotService from '#services/SpotService' + +export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name' + +export interface SearchParams { + q?: string + /** Only releases whose name is readable (not an obfuscated token). */ + readable?: boolean + /** Only releases that have all their files. */ + complete?: boolean + /** Hide password-protected releases. */ + hideEncrypted?: boolean + source?: string + poster?: string + sort?: SortOrder + page?: number + perPage?: number + /** Newznab: exact category ids (already expanded from parents). */ + categories?: number[] + /** Newznab: leave out releases known to be missing files. */ + notIncomplete?: boolean + maxAgeDays?: number + /** TV: season and episode numbers, matched as S01E02 or 1x02 in the name. */ + season?: number + episode?: number + /** Overrides `page` (Newznab uses offsets). */ + offset?: number + /** Leave out releases that are only an encrypted container (nzbe). */ + hideContainers?: boolean + /** Include incomplete releases that consist of par2 files only (hidden by default). */ + includePar2Only?: boolean +} + +const SORTS: Record = { + newest: 'r.posted_at DESC NULLS LAST, r.id DESC', + oldest: 'r.posted_at ASC NULLS LAST, r.id ASC', + largest: 'r.size DESC, r.id DESC', + files: 'r.file_count DESC, r.id DESC', + name: 'r.name ASC, r.id ASC', +} + +/** + * Shown in search: releases with files, and Spotnet listings whose NZB is + * fetched on first download (CONCEPT.md D29). Matches the partial indexes. + */ +const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')` + +/** Incomplete, and every file is a par2 file: nothing to download but repair data. */ +const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS ( + SELECT 1 FROM files f WHERE f.release_id = r.id + AND f.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))` + +/** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */ +function pattern(word: string): string { + const escaped = word.replace(/[\\%]/g, (c) => `\\${c}`) + return `%${escaped.replace(/[._\- ]+/g, '_')}%` +} + +/** + * Searching and browsing releases for the web interface (CONCEPT.md D22). + */ +class SearchService { + async search(params: SearchParams) { + const where: string[] = [ + VISIBLE_SQL, + // Retracted by its poster or a Spotnet moderator (CONCEPT.md D29). + `NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status = 'removed')`, + ] + const bindings: unknown[] = [] + + const words = (params.q ?? '').trim().split(/\s+/).filter(Boolean).slice(0, 8) + for (const word of words) { + const p = pattern(word) + // Every word must match the release name, a file name, or the archive contents. + // One trigram-indexed lookup per table, not a subquery per release. + where.push(`(r.name ILIKE ? OR r.contents::text ILIKE ? + OR r.id IN (SELECT f.release_id FROM files f WHERE f.filename ILIKE ? OR f.real_name ILIKE ?))`) + bindings.push(p, p, p, p) + } + if (params.readable) where.push('is_readable_name(r.name)') + if (params.hideEncrypted) where.push('r.encrypted IS NOT TRUE') + if (params.hideContainers) where.push('r.container IS NULL') + // par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch). + if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`) + // Kept current by triggers (migration …860). + if (params.complete) where.push('r.complete') + if (params.source) { + where.push('r.source = ?') + bindings.push(params.source) + } + if (params.poster) { + where.push('r.poster ILIKE ?') + bindings.push(pattern(params.poster)) + } + if (params.categories?.length) { + where.push('r.category = ANY(?)') + bindings.push(params.categories) + } + if (params.notIncomplete) { + where.push(`(r.complete OR (r.file_total IS NULL AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)))`) + } + if (params.maxAgeDays) { + where.push(`r.posted_at >= now() - make_interval(days => ?)`) + bindings.push(params.maxAgeDays) + } + if (params.season !== undefined) { + const s = params.season + const e = params.episode + where.push('r.name ~* ?') + bindings.push( + e !== undefined + ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` + : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)` + ) + } + + const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200) + const page = Math.max(params.page ?? 1, 1) + const offset = params.offset ?? (page - 1) * perPage + const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest + const whereSql = where.join(' AND ') + + const [{ rows }, { rows: countRows }] = await Promise.all([ + db.rawQuery( + `SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.file_count, r.category, r.container, + r.size, r.encrypted, is_readable_name(r.name) AS readable, + (SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id) AS listed_files, + (SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id AND EXISTS ( + SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id + WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present, + jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries, + r.complete, ${PAR2_ONLY_SQL} AS par2_only + FROM releases r WHERE ${whereSql} + ORDER BY ${order} LIMIT ? OFFSET ?`, + [...bindings, perPage, offset] + ), + db.rawQuery(`SELECT count(*) AS total FROM releases r WHERE ${whereSql}`, bindings), + ]) + + return { + page, + perPage, + offset, + total: Number(countRows[0].total), + releases: rows.map((r: any) => ({ + id: Number(r.id), + name: r.name, + source: r.source, + poster: r.poster, + groups: r.groups, + postedAt: r.posted_at, + fileTotal: r.file_total, + fileCount: r.file_count, + size: Number(r.size), + encrypted: r.encrypted, + readable: r.readable, + listedFiles: Number(r.listed_files), + listedPresent: Number(r.listed_present), + archiveEntries: Number(r.archive_entries), + complete: r.complete, + category: r.category, + container: r.container, + par2Only: r.par2_only, + })), + } + } + + async release(id: number) { + const release = await db.from('releases').where('id', id).first() + if (!release) return null + + const [files, listed, pending] = await Promise.all([ + db + .from('files as f') + .leftJoin('binaries as b', 'b.file_id', 'f.id') + .where('f.release_id', id) + .orderByRaw('coalesce(f.real_name, f.filename)') + .select('f.id', 'f.filename', 'f.real_name', 'f.size', 'f.parts', 'f.date', 'f.poster', 'b.size as exact_size'), + db.rawQuery( + `SELECT rf.name, rf.size, + EXISTS (SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id + WHERE f.release_id = rf.release_id AND (f.real_name = rf.name OR b.size = rf.size)) AS present + FROM release_files rf WHERE rf.release_id = ? ORDER BY rf.name`, + [id] + ), + db.rawQuery( + `SELECT count(*) AS binaries, coalesce(sum(b.total_parts), 0) AS parts_total, + coalesce(sum((SELECT count(*) FROM parts p WHERE p.binary_id = b.id)), 0) AS parts_have + FROM binaries b WHERE b.release_id = ? AND b.completed_at IS NULL AND b.conflicted_at IS NULL`, + [id] + ), + ]) + + const spot = release.source === 'spot' + ? await db.from('spots').where('release_id', id).select('tag', 'poster', 'category', 'website').first() + : null + return { + id: Number(release.id), + name: release.name, + source: release.source, + spot: spot ? { tag: spot.tag, poster: spot.poster, category: spot.category, website: spot.website } : null, + listingOnly: release.source === 'spot' && Number(release.file_count) === 0, + poster: release.poster, + groups: release.groups, + postedAt: release.posted_at, + fileTotal: release.file_total, + fileCount: release.file_count, + size: Number(release.size), + encrypted: release.encrypted, + container: release.container, + contents: release.contents ?? [], + files: files.map((f) => ({ + id: Number(f.id), + name: f.real_name ?? f.filename, + postedName: f.filename, + realName: f.real_name, + size: Number(f.exact_size ?? f.size), + parts: f.parts, + postedAt: f.date ? new Date(Number(f.date)) : null, + poster: f.poster, + })), + par2Files: listed.rows.map((r: any) => ({ name: r.name, size: Number(r.size), present: r.present })), + pending: { + binaries: Number(pending.rows[0].binaries), + partsHave: Number(pending.rows[0].parts_have), + partsTotal: Number(pending.rows[0].parts_total), + }, + } + } + + async nzb(id: number): Promise<{ name: string; xml: string } | null> { + let release = await db.from('releases').where('id', id).first() + if (!release) return null + if (release.source === 'spot' && Number(release.file_count) === 0) { + // A Spotnet listing: read its NZB from Usenet now (the import may join known files elsewhere). + const imported = await SpotService.importListing(id) + if (!imported) return null + id = imported + release = await db.from('releases').where('id', id).first() + } + const files = await db + .from('files') + .where('release_id', id) + .orderByRaw('coalesce(real_name, filename)') + .select('filename', 'real_name', 'poster', 'date', 'groups', 'message_ids') + const xml = buildNzb( + files.map((f) => ({ + name: f.real_name ?? f.filename, + poster: f.poster, + date: Number(f.date), + groups: f.groups?.length ? f.groups : ['alt.binaries.test'], + segments: f.message_ids, + })), + { title: release.name, password: release.password ?? undefined } + ) + return { name: release.name, xml } + } + + async stats() { + const { rows } = await db.rawQuery(` + SELECT (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL}) AS releases, + (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL} AND is_readable_name(name)) AS readable_releases, + (SELECT count(*) FROM files) AS files, + (SELECT coalesce(sum(size), 0) FROM files) AS size, + (SELECT count(*) FROM releases WHERE encrypted) AS encrypted_releases, + (SELECT max(updated_at) FROM groups) AS last_indexed_at, + (SELECT json_agg(json_build_object('source', source, 'releases', n) ORDER BY n DESC) + FROM (SELECT coalesce(source, 'unknown') AS source, count(*) AS n FROM releases r WHERE ${VISIBLE_SQL} GROUP BY 1) s) AS sources`) + const r = rows[0] + return { + releases: Number(r.releases), + readableReleases: Number(r.readable_releases), + files: Number(r.files), + size: Number(r.size), + encryptedReleases: Number(r.encrypted_releases), + lastIndexedAt: r.last_indexed_at, + sources: r.sources ?? [], + } + } +} + +export default new SearchService() diff --git a/app/services/Shutdown.ts b/app/services/Shutdown.ts new file mode 100644 index 0000000..ecf4032 --- /dev/null +++ b/app/services/Shutdown.ts @@ -0,0 +1,45 @@ +import { setTimeout as sleep } from 'node:timers/promises' + +/** After a signal, give cleanup this long before exiting anyway. */ +const FORCE_EXIT_MS = 25_000 + +/** + * Graceful shutdown for the long-running commands. SIGTERM (docker stop) and + * SIGINT wake up `wait()` and `sleep()`, so workers can finish their active + * jobs and close their connections before the process exits. + */ +class Shutdown { + private readonly controller = new AbortController() + + constructor() { + for (const signal of ['SIGTERM', 'SIGINT'] as const) { + process.once(signal, () => { + this.controller.abort() + setTimeout(() => process.exit(0), FORCE_EXIT_MS).unref() + }) + } + } + + get requested(): boolean { + return this.controller.signal.aborted + } + + /** Resolves when a shutdown signal arrives. */ + wait(): Promise { + if (this.requested) return Promise.resolve() + return new Promise((resolve) => + this.controller.signal.addEventListener('abort', () => resolve(), { once: true }) + ) + } + + /** Sleeps, but returns early on shutdown. */ + async sleep(ms: number): Promise { + try { + await sleep(ms, undefined, { signal: this.controller.signal }) + } catch { + // aborted by a shutdown signal + } + } +} + +export default new Shutdown() diff --git a/app/services/SplitService.ts b/app/services/SplitService.ts new file mode 100644 index 0000000..7c32186 --- /dev/null +++ b/app/services/SplitService.ts @@ -0,0 +1,146 @@ +import { createHash } from 'node:crypto' +import db from '@adonisjs/lucid/services/db' +import IngestService from '#services/IngestService' +import NameService from '#services/NameService' +import { splitVerified } from '#lib/split/index' + +/** Conflicted binaries handled per run; the rest follow on the next tick. */ +const BATCH = 200 + +export interface SplitResult { + parents: number + volumes: number + completed: number +} + +/** + * Splits conflicted obfuscated binaries (several equal-sized files under one + * size+parts identity) into one binary per volume. Only volumes whose + * combined part CRCs match the file CRC of their last part are created, so a + * split can never produce a corrupt file; the rest waits for more articles. + * See CONCEPT.md (D20). + */ +class SplitService { + async splitPending(): Promise { + const parents = await db + .from('binaries') + .whereNotNull('conflicted_at') + .whereNull('parent_binary_id') + // split_at = 'infinity' marks binaries that cannot be split (see migration …810). + .where((q) => q.whereNull('split_at').orWhereRaw('last_seen_at > split_at')) + // Most recently active first: those are the posts still coming in. + .orderBy('last_seen_at', 'desc') + .limit(BATCH) + + const volumeIds: number[] = [] + for (const parent of parents) { + volumeIds.push(...(await this.split(parent))) + } + + // Former conflicts that turned out to be reposts (unflagged, keep their + // split_at): no new article may arrive to trigger their completion. + const unflagged = await db + .from('binaries') + .whereNull('conflicted_at') + .whereNull('completed_at') + .whereNull('parent_binary_id') + .whereNotNull('split_at') + .whereRaw("split_at <> 'infinity'") + .limit(BATCH) + .select('id') + volumeIds.push(...unflagged.map((b) => Number(b.id))) + + await IngestService.matchPar2Releases(volumeIds) + await NameService.resolve(volumeIds) + const completed = await IngestService.claimCompleted(volumeIds) + await IngestService.enqueue(completed) + return { parents: parents.length, volumes: volumeIds.length, completed: completed.length } + } + + /** Creates the newly verified volumes of one conflicted binary; returns their ids. */ + private async split(parent: any): Promise { + return db.transaction(async (trx) => { + // Unverified volumes from before verification existed can never complete. + await trx.from('binaries').where('parent_binary_id', parent.id).whereNull('completed_at').delete() + + const { rows } = await trx.rawQuery( + `WITH taken AS ( + SELECT p.message_id FROM parts p JOIN binaries v ON v.id = p.binary_id + WHERE v.parent_binary_id = ? + ), candidates AS ( + SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM parts WHERE binary_id = ? + UNION ALL + SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM part_conflicts WHERE binary_id = ? + ) + SELECT * FROM candidates WHERE message_id NOT IN (SELECT message_id FROM taken)`, + [parent.id, parent.id, parent.id] + ) + const volumes = splitVerified( + rows.map((r: any) => ({ + number: r.number, + messageId: r.message_id, + bytes: r.bytes, + articleNumber: r.article_number === null ? null : Number(r.article_number), + pcrc32: r.pcrc32, + partSize: r.part_size, + fileCrc32: r.file_crc32, + })), + { totalParts: parent.total_parts } + ) + + const now = new Date() + const ids: number[] = [] + for (const articles of volumes) { + // A verified volume is fixed, so its first article identifies it. + const fileKey = createHash('sha1').update(`${parent.file_key}#${articles[0].messageId}`).digest('hex') + const inserted = await trx + .knexClient('binaries') + .insert({ + file_key: fileKey, + poster: parent.poster, + subject: parent.subject, + name: parent.name, + total_parts: parent.total_parts, + groups: JSON.stringify(parent.groups), + posted_at: parent.posted_at, + size: parent.size, + release_id: parent.release_id, + parent_binary_id: parent.id, + first_seen_at: now, + last_seen_at: now, + }) + .onConflict('file_key') + .ignore() + .returning('id') + if (inserted.length === 0) continue + const id = Number(inserted[0].id) + await trx.knexClient('parts').insert( + articles.map((a) => ({ + binary_id: id, + number: a.number, + message_id: a.messageId, + bytes: a.bytes, + article_number: a.articleNumber, + pcrc32: a.pcrc32, + part_size: a.partSize, + file_crc32: a.fileCrc32, + })) + ) + // The fingerprint of a volume is that of its part 1 article. + await trx.rawQuery( + `UPDATE binaries SET md5_16k = ( + SELECT md5_16k FROM part_conflicts WHERE binary_id = ? AND message_id = ? + UNION ALL SELECT md5_16k FROM parts WHERE binary_id = ? AND message_id = ? LIMIT 1) + WHERE id = ?`, + [parent.id, articles[0].messageId, parent.id, articles[0].messageId, id] + ) + ids.push(id) + } + + await trx.from('binaries').where('id', parent.id).update({ split_at: now }) + return ids + }) + } +} + +export default new SplitService() diff --git a/app/services/SpotService.ts b/app/services/SpotService.ts new file mode 100644 index 0000000..bff6916 --- /dev/null +++ b/app/services/SpotService.ts @@ -0,0 +1,172 @@ +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import indexerConfig from '#config/indexer' +import NntpService from '#services/NntpService' +import NzbImportService from '#services/NzbImportService' +import { type OverviewHeader } from '#services/IngestService' +import { parseNzb } from '#lib/nzb/index' +import { + moderationTarget, + spotterId, + decodeSpotBinary, + parseSpotArticleHeaders, + parseSpotHeader, + parseSpotXml, + verifySpotHeader, + type SpotHeader, +} from '#lib/spotnet/index' + +/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */ +const MAX_NZB_SEGMENTS = 20 +/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */ +const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60 +/** A claimed spot whose worker died is taken again after this long. */ +const RECLAIM_AFTER = '1 hour' + +export function isSpotGroup(name: string): boolean { + return indexerConfig.spotGroups.includes(name) +} + +/** + * Spotnet (CONCEPT.md D29). Spots are signed posts in free.pt; their full + * header holds XML with the message ids of a compressed NZB, which is + * imported like a posted NZB (D26). Spot NZBs name many obfuscated posts + * (nyuu, ngPost, the skipped multi-group uploader). + */ +class SpotService { + /** Queues the spots among a batch of one spot group's headers. Returns how many. */ + async recordHeaders(groupName: string, overviews: [number, OverviewHeader][]): Promise { + const now = new Date() + const rows = [] + for (const [articleNumber, header] of overviews) { + if (!header?.from || !header['message-id']) continue + const spot = parseSpotHeader(header.from, header.subject ?? '', header['message-id'], header.date) + if (!spot?.title) continue + rows.push({ + message_id: spot.messageId, + group_name: groupName, + article_number: Number(articleNumber), + title: spot.title, + tag: spot.tag || null, + poster: spot.poster, + category: spot.category, + subcats: JSON.stringify(spot.subcats), + size: spot.size || null, + posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null, + key_id: spot.keyId, + spotter_id: spotterId(spot.selfSignedPubKey) || null, + created_at: now, + }) + } + if (rows.length === 0) return 0 + await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore() + return rows.length + } + + /** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */ + async claim(limit: number): Promise { + const { rows } = await db.rawQuery( + `UPDATE spots SET status = 'processing', processed_at = now() + WHERE message_id IN ( + SELECT message_id FROM spots + WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}') + ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED) + RETURNING *`, + [limit] + ) + return rows + } + + /** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */ + async process(spot: any): Promise { + const update: Record = { processed_at: new Date(), error: null } + try { + const raw = await this.fetch(`HEAD <${spot.message_id}>`) + const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1')) + // The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled. + const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date) + if (!header || !verifySpotHeader(header)) { + update.status = 'unverified' + } else if (moderationTarget(header)) { + update.status = 'moderation' + await this.moderate(header, moderationTarget(header)!) + } else { + const parsed = parseSpotXml(xml) + update.title = header.title + update.poster = header.poster + update.spotter_id = spotterId(header.selfSignedPubKey) || null + if (parsed?.website) update.website = parsed.website + const segments = parsed?.nzbSegments ?? [] + if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) { + update.status = 'no-nzb' + } else { + const bodies = [] + for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`)) + const nzbXml = decodeSpotBinary(bodies).toString('utf8') + const nzb = parseNzb(nzbXml) + if (!nzb || nzb.files.length === 0) { + update.status = 'no-nzb' + } else { + const groups = [...new Set(nzb.files.flatMap((f) => f.groups))] + const result = await NzbImportService.import( + { + id: null, + filename: `${header.title}.nzb`, + poster: header.poster, + groups, + date: header.stamp * 1000, + message_ids: { '1': { id: spot.message_id } }, + }, + nzbXml, + { name: header.title, source: 'spot', moveContainer: false } + ) + update.status = result ? 'imported' : 'no-nzb' + if (result) update.release_id = result.releaseId + } + } + } + } catch (error: any) { + update.status = 'failed' + update.error = String(error.message).slice(0, 500) + logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`) + } + await db.from('spots').where('message_id', spot.message_id).update(update) + return update.status as string + } + + /** + * Applies a delete/dispose/remove: the target spot is marked `removed` + * (its release is then hidden from search). As in Spotweb, a personal + * dispose must come from the same spotter within 5 days; a moderator's + * (signed with Spotnet's key, no personal key) always counts. + */ + private async moderate(header: SpotHeader, targetId: string) { + const target = await db.from('spots').where('message_id', targetId).first() + if (!target) return + const personal = header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : '' + if (personal) { + const age = header.stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0) + if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return + } + await db.from('spots').where('message_id', targetId).update({ status: 'removed' }) + logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`) + } + + /** One HEAD or BODY, as raw bytes. */ + private async fetch(command: string): Promise { + const conn = await NntpService.acquire() + let failed = false + try { + const { data } = await conn._artcmd(command) + return Buffer.from(data.buffer, data.byteOffset, data.byteLength) + } catch (error: any) { + // 430 (no such article) is an answer, not a broken connection. + failed = !/^4\d\d/.test(String(error.message)) + throw error + } finally { + NntpService.release(conn, failed) + } + } +} + +export default new SpotService() diff --git a/app/services/YencFile.ts b/app/services/YencFile.ts deleted file mode 100644 index e42c795..0000000 --- a/app/services/YencFile.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { decode } from 'simple-yenc' - -export class YencFile { - private buffer: Buffer | null = null - - public processPart(partBuffer: Buffer) { - // This is a simplified implementation. - // simple-yenc's decode function is synchronous and works on a full buffer. - // A more complex implementation would handle multi-part decoding. - this.buffer = decode(partBuffer) - } - - public getBuffer(): Buffer { - if (!this.buffer) { - throw new Error('No data has been processed yet.') - } - return this.buffer - } -} diff --git a/app/services/YencService.ts b/app/services/YencService.ts deleted file mode 100644 index ca0e01a..0000000 --- a/app/services/YencService.ts +++ /dev/null @@ -1,36 +0,0 @@ -export function parseYencMeta(buffer: Buffer): { header: Record; crc32?: string } { - const text = buffer.toString('latin1') - const lines = text.split(/\\r?\\n/) - - const header: Record = {} - let crc32: string | undefined - - for (const line of lines) { - if (line.startsWith('=ybegin')) { - const parts = line.split(' ') - parts.forEach((part) => { - if (part.includes('=')) { - const [key, value] = part.split('=') - if (key === 'name') { - header[key] = value.trim() - } else { - header[key] = parseInt(value, 10) - } - } - }) - } else if (line.startsWith('=ypart')) { - const match = /begin=(\d+)/.exec(line) - if (match) { - header.partBegin = parseInt(match[1], 10) - } - } else if (line.startsWith('=yend')) { - const match = /crc32=([a-fA-F0-9]+)/.exec(line) - if (match) { - crc32 = match[1] - } - break // End of yenc data - } - } - - return { header, crc32 } -} diff --git a/commands/BodyQueuePrune.ts b/commands/BodyQueuePrune.ts new file mode 100644 index 0000000..d97dcb7 --- /dev/null +++ b/commands/BodyQueuePrune.ts @@ -0,0 +1,97 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import indexerConfig from '#config/indexer' +import QueueService from '#services/QueueService' +import RedisService from '#services/RedisService' +import DeferredService, { isDeferred } from '#services/DeferredService' + +const BATCH = 2000 + +/** Rebuilds the wait list (KEYS[1]) without the ids in the set KEYS[2], in one atomic pass. */ +const FILTER_LIST = ` +local ids = redis.call('LRANGE', KEYS[1], 0, -1) +local keep = {} +for _, id in ipairs(ids) do + if redis.call('SISMEMBER', KEYS[2], id) == 0 then keep[#keep + 1] = id end +end +redis.call('DEL', KEYS[1]) +for i = 1, #keep, 5000 do + redis.call('RPUSH', KEYS[1], unpack(keep, i, math.min(i + 4999, #keep))) +end +return #ids - #keep` + +/** + * Removes waiting body jobs whose message-id is on the skip or defer list, + * e.g. after adding a pattern (CONCEPT.md D24, D28). Deferred ones are + * recorded in deferred_files first, so they can still be processed later. + */ +export default class BodyQueuePrune extends BaseCommand { + public static commandName = 'body-queue:prune' + public static description = 'Drops waiting body jobs that are now skipped or deferred.' + + public static options: CommandOptions = { + startApp: true, + } + + @flags.boolean({ description: 'Only count' }) + declare dryRun: boolean + + @flags.boolean({ description: 'Drop deferred jobs without recording them (already recorded by an earlier run)' }) + declare skipRecord: boolean + + public async run() { + const redis = RedisService.client + const queue = QueueService.bodyQueue + const waitKey = queue.toKey('wait') + let skipped = 0 + let deferred = 0 + let scanned = 0 + const dropped: string[] = [] + try { + const ids: string[] = await redis.lrange(waitKey, 0, -1) + for (let i = 0; i < ids.length; i += BATCH) { + const batch = ids.slice(i, i + BATCH) + const data: [Error | null, string | null][] = await batch + .reduce((pipeline, id) => pipeline.hget(queue.toKey(id), 'data'), redis.pipeline()) + .exec() + const drop: string[] = [] + const toDefer = new Map }[]>() + batch.forEach((id, j) => { + const job = data[j][1] ? JSON.parse(data[j][1]!) : null + if (!job?.header) return + const messageId = job.header['message-id'] ?? '' + if (isDeferred(job.header)) { + const list = toDefer.get(job.group) ?? [] + list.push({ articleNumber: Number(job.articleNumber), header: job.header }) + toDefer.set(job.group, list) + drop.push(id) + deferred++ + } else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(messageId))) { + drop.push(id) + skipped++ + } + }) + scanned += batch.length + if (this.dryRun || drop.length === 0) continue + if (!this.skipRecord) for (const [group, articles] of toDefer) await DeferredService.record(group, articles) + dropped.push(...drop) + } + if (dropped.length > 0) { + const dropKey = queue.toKey('prune-ids') + await redis.del(dropKey) + for (let i = 0; i < dropped.length; i += BATCH) await redis.sadd(dropKey, ...dropped.slice(i, i + BATCH)) + await redis.eval(FILTER_LIST, 2, waitKey, dropKey) + await redis.del(dropKey) + for (let i = 0; i < dropped.length; i += BATCH) { + await redis.unlink(...dropped.slice(i, i + BATCH).map((id) => queue.toKey(id))) + } + } + this.logger.info( + `Scanned ${scanned} waiting body jobs: ${this.dryRun ? 'would drop' : 'dropped'} ${deferred} deferred and ${skipped} skipped.` + ) + } finally { + await QueueService.closeAll() + await RedisService.close() + } + } +} diff --git a/commands/ContentBackfill.ts b/commands/ContentBackfill.ts new file mode 100644 index 0000000..e49d7b0 --- /dev/null +++ b/commands/ContentBackfill.ts @@ -0,0 +1,68 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import db from '@adonisjs/lucid/services/db' +import QueueService from '#services/QueueService' +import ContentService from '#services/ContentService' + +const BATCH = 1000 + +/** + * Queues completed files for content inspection (names and groupings from + * par2 and RAR headers, CONCEPT.md D21) that were completed before it + * existed. Costs one article download per file, done by worker:collection. + */ +export default class ContentBackfill extends BaseCommand { + public static commandName = 'content:backfill' + public static description = 'Queues existing files with random names for content inspection.' + + public static options: CommandOptions = { + startApp: true, + } + + @flags.number({ description: 'Queue at most this many files' }) + declare limit?: number + + @flags.boolean({ description: 'Also files that were inspected before (after release:rebuild --reset-content)' }) + declare all: boolean + + @flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' }) + declare archiveNzbs: boolean + + public async run() { + let lastId = 0 + let queued = 0 + try { + while (!this.limit || queued < this.limit) { + // Files never inspected: no fingerprint on their binary and no archive listing. + const query = db + .from('files as f') + .join('binaries as b', 'b.file_id', 'f.id') + .where('f.id', '>', lastId) + if (this.archiveNzbs) query.whereRaw(`f.archive::text ~* '\\.nzb"'`) + else if (!this.all) query.whereNull('b.md5_16k').whereNull('f.archive') + const files = await query + .orderBy('f.id') + .limit(BATCH) + .select('f.id', 'f.filename') + if (files.length === 0) break + lastId = Number(files.at(-1).id) + + const jobs = files + .filter((f) => ContentService.isCandidate(f.filename)) + .slice(0, this.limit ? this.limit - queued : undefined) + .map((f) => ({ + name: 'process-collection', + data: { fileId: Number(f.id) }, + // A new id per run: BullMQ ignores an add() for an id it still knows. + // Targeted backfills go ahead of whatever is waiting. + opts: { jobId: `content-${f.id}-${Date.now()}`, lifo: this.archiveNzbs }, + })) + if (jobs.length > 0) await QueueService.collectionQueue.addBulk(jobs) + queued += jobs.length + } + this.logger.info(`Queued ${queued} files for content inspection.`) + } finally { + await QueueService.closeAll() + } + } +} diff --git a/commands/DeferredExpandRanges.ts b/commands/DeferredExpandRanges.ts new file mode 100644 index 0000000..3d80ca8 --- /dev/null +++ b/commands/DeferredExpandRanges.ts @@ -0,0 +1,32 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import NntpService from '#services/NntpService' +import QueueService from '#services/QueueService' +import DeferredService from '#services/DeferredService' + +/** + * Sends recorded deferred ranges (fully obfuscated ngPost articles, CONCEPT.md + * D28) to the body queue, oldest first. Every article in them is downloaded: + * their headers can't be grouped into files. + */ +export default class DeferredExpandRanges extends BaseCommand { + public static commandName = 'deferred:expand-ranges' + public static description = 'Queues the bodies of deferred article ranges (costly: every article is downloaded).' + + public static options: CommandOptions = { + startApp: true, + } + + @flags.number({ description: 'Ranges to expand', default: 10 }) + declare limit: number + + public async run() { + try { + const { ranges, articles } = await DeferredService.expandRanges(this.limit) + this.logger.info(`Expanded ${ranges} ranges: ${articles} articles sent to the body queue.`) + } finally { + await NntpService.shutdown() + await QueueService.closeAll() + } + } +} diff --git a/commands/GcScheduler.ts b/commands/GcScheduler.ts new file mode 100644 index 0000000..6582f8b --- /dev/null +++ b/commands/GcScheduler.ts @@ -0,0 +1,282 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import type { Queue } from 'bullmq' +import db from '@adonisjs/lucid/services/db' +import gcConfig from '#config/gc' +import QueueService from '#services/QueueService' +import RedisService from '#services/RedisService' +import Shutdown from '#services/Shutdown' + +const CLEAN_BATCH = 1000 +const SCAN_COUNT = 1000 + +export default class GcScheduler extends BaseCommand { + public static commandName = 'gc:scheduler' + public static description = 'Periodically removes old BullMQ jobs and stale binaries.' + + public static options: CommandOptions = { + startApp: true, + } + + @flags.boolean({ description: 'Run one sweep and exit' }) + declare once: boolean + + @flags.boolean({ description: 'Report what would be removed without changing anything' }) + declare dryRun: boolean + + public async run() { + this.logger.info( + `GC started (interval ${gcConfig.intervalMs / 1000}s${this.dryRun ? ', dry run' : ''}).` + ) + + try { + do { + try { + await this.sweep() + } catch (err: any) { + this.logger.error(`GC sweep failed: ${err.message}`) + } + if (!this.once) await Shutdown.sleep(gcConfig.intervalMs) + } while (!this.once && !Shutdown.requested) + } finally { + await QueueService.closeAll() + await RedisService.close() + } + } + + private async sweep() { + const { completedAgeSeconds, failedAgeSeconds } = gcConfig.jobs + for (const queue of [...QueueService.all, ...QueueService.legacy]) { + const completed = await this.cleanJobs(queue, 'completed', completedAgeSeconds) + const failed = await this.cleanJobs(queue, 'failed', failedAgeSeconds) + if (completed || failed) { + this.logger.info( + `${queue.name}: ${this.verb('remove', 'removed')} ${completed} completed and ${failed} failed jobs.` + ) + } + } + + const trimmed = await this.trimBodyQueue() + if (trimmed) this.logger.info(`Body queue: ${this.verb('drop', 'dropped')} ${trimmed} oldest waiting jobs.`) + + const binaries = await this.deleteStaleBinaries() + this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`) + + if (!this.dryRun) { + const budget = Date.now() + gcConfig.deleteBudgetMs + const parts = await this.dropCopiedParts(budget) + const single = await this.deleteSingleArticleBinaries(budget) + const releases = await this.deleteEmptyReleases(budget) + if (parts || single || releases) { + this.logger.info( + `Space: ${parts} copied parts, ${single} single-article binaries, ${releases} empty releases deleted.` + ) + } + } + + const deferred = await this.deleteOldDeferredFiles() + if (deferred) this.logger.info(`Deferred files: ${this.verb('delete', 'deleted')} ${deferred} past retention.`) + + const { expired, scanned } = await this.expireLegacyPendingFiles() + if (scanned > 0) { + this.logger.info( + `Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.` + ) + } + } + + /** + * Drops the oldest waiting body jobs above gcConfig.bodyQueueMax. BullMQ + * workers take jobs from the right end of the wait list, so the oldest are + * there. + */ + private async trimBodyQueue(): Promise { + const redis = RedisService.client + const key = QueueService.bodyQueue.toKey('wait') + const excess = (await redis.llen(key)) - gcConfig.bodyQueueMax + if (excess <= 0 || this.dryRun) return Math.max(excess, 0) + let dropped = 0 + while (dropped < excess) { + const ids: string[] = (await redis.rpop(key, Math.min(5000, excess - dropped))) ?? [] + if (ids.length === 0) break + await redis.unlink(...ids.map((id) => QueueService.bodyQueue.toKey(id))) + dropped += ids.length + } + return dropped + } + + /** + * Deletes binaries (and, by cascade, their parts) that received no segment + * for `staleAfterSeconds`: incomplete ones are given up on; completed ones + * already live on in `files`; conflicted ones will never become a file. + * Completed ones still waiting for worker:file are kept. + */ + private async deleteStaleBinaries(): Promise { + const query = db + .from('binaries') + .where('last_seen_at', '<', new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000)) + .where((q) => + q.whereNull('completed_at').orWhereNotNull('file_id').orWhereNotNull('conflicted_at') + ) + + if (this.dryRun) { + const [{ count }] = await query.count('* as count') + return Number(count) + } + return Number(await query.delete()) + } + + private async deleteOldDeferredFiles(): Promise { + const cutoff = new Date(Date.now() - gcConfig.deferredRetentionDays * 24 * 60 * 60 * 1000) + let total = 0 + for (const [table, column] of [['deferred_files', 'first_seen_at'], ['deferred_ranges', 'created_at']]) { + const query = db.from(table).where(column, '<', cutoff) + if (this.dryRun) { + const [{ count }] = await query.count('* as count') + total += Number(count) + } else { + total += Number(await query.delete()) + } + } + return total + } + + // Keyset cursors over the tables, kept between sweeps so one pass over + // millions of rows is spread over several sweeps. + private singleCursor = 0 + private releaseCursor = 0 + + /** Parts of header-derived binaries that are a file already (worker:file now drops them itself). */ + private async dropCopiedParts(budget: number): Promise { + let total = 0 + while (Date.now() < budget) { + const result = await db.rawQuery( + `DELETE FROM parts WHERE binary_id IN ( + SELECT b.id FROM binaries b + WHERE b.file_id IS NOT NULL AND b.size IS NULL + AND EXISTS (SELECT 1 FROM parts p WHERE p.binary_id = b.id) + LIMIT ?)`, + [500] + ) + total += result.rowCount ?? 0 + if (!result.rowCount) break + } + return total + } + + /** Header-derived binaries that never got a second article (gcConfig.binaries.singleArticleAfterSeconds). */ + private async deleteSingleArticleBinaries(budget: number): Promise { + const cutoff = new Date(Date.now() - gcConfig.binaries.singleArticleAfterSeconds * 1000) + let total = 0 + while (Date.now() < budget) { + const { rows } = await db.rawQuery( + `WITH batch AS ( + SELECT id FROM binaries WHERE id > ? ORDER BY id LIMIT ? + ), gone AS ( + DELETE FROM binaries b USING batch + WHERE b.id = batch.id AND b.completed_at IS NULL AND b.size IS NULL AND b.total_parts > 1 + AND b.last_seen_at < ? AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) <= 1 + RETURNING 1) + SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`, + [this.singleCursor, gcConfig.deleteBatch, cutoff] + ) + total += Number(rows[0].deleted) + if (rows[0].last === null) { + this.singleCursor = 0 + break + } + this.singleCursor = Number(rows[0].last) + } + return total + } + + /** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */ + private async deleteEmptyReleases(budget: number): Promise { + const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000) + let total = 0 + while (Date.now() < budget) { + const { rows } = await db.rawQuery( + `WITH batch AS ( + SELECT id FROM releases WHERE id > ? ORDER BY id LIMIT ? + ), gone AS ( + DELETE FROM releases r USING batch + WHERE r.id = batch.id AND r.file_count = 0 AND r.first_seen_at < ? + AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id) + AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id) + AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id) + AND NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id) + RETURNING 1) + SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`, + [this.releaseCursor, gcConfig.deleteBatch, cutoff] + ) + total += Number(rows[0].deleted) + if (rows[0].last === null) { + this.releaseCursor = 0 + break + } + this.releaseCursor = Number(rows[0].last) + } + return total + } + + /** Removes jobs in `state` that finished more than `ageSeconds` ago. */ + private async cleanJobs(queue: Queue, state: 'completed' | 'failed', ageSeconds: number) { + const graceMs = ageSeconds * 1000 + if (this.dryRun) { + const cutoff = Date.now() - graceMs + // Finished-job sets are sorted by finishedOn, so a score range counts them cheaply. + return RedisService.client.zcount(queue.toKey(state), '-inf', cutoff) + } + + let total = 0 + let removed: string[] + do { + removed = await queue.clean(graceMs, CLEAN_BATCH, state) + total += removed.length + } while (removed.length === CLEAN_BATCH) + return total + } + + /** + * Pending-file hashes of the old header worker (and the pre-AdonisJS app) + * never expire. Give them a TTL rather than deleting them outright. + */ + private async expireLegacyPendingFiles() { + const redis = RedisService.client + const { prefix, ttlSeconds } = gcConfig.legacyPendingFiles + let cursor = '0' + let scanned = 0 + let expired = 0 + + do { + const [next, keys]: [string, string[]] = await redis.scan( + cursor, + 'MATCH', + `${prefix}*`, + 'COUNT', + SCAN_COUNT + ) + cursor = next + if (keys.length === 0) continue + scanned += keys.length + + const ttls: [Error | null, number][] = await keys + .reduce((pipeline, key) => pipeline.ttl(key), redis.pipeline()) + .exec() + const orphaned = keys.filter((_, i) => ttls[i][1] === -1) + expired += orphaned.length + + if (!this.dryRun && orphaned.length > 0) { + await orphaned + .reduce((pipeline, key) => pipeline.expire(key, ttlSeconds, 'NX'), redis.pipeline()) + .exec() + } + } while (cursor !== '0') + + return { scanned, expired } + } + + private verb(present: string, past: string) { + return this.dryRun ? `would ${present}` : past + } +} diff --git a/commands/GroupAdd.ts b/commands/GroupAdd.ts new file mode 100644 index 0000000..85f851c --- /dev/null +++ b/commands/GroupAdd.ts @@ -0,0 +1,72 @@ +import { BaseCommand, args, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import Group from '#models/group' +import NntpService from '#services/NntpService' +import { parseGroupResponse } from '#lib/nntp/group' + +/** + * Adds newsgroups to index (or reactivates them). The scheduler picks them + * up on its next tick, starting INDEX_INITIAL_BACKFILL articles back. + */ +export default class GroupAdd extends BaseCommand { + public static commandName = 'group:add' + public static description = 'Adds newsgroups to index.' + + public static options: CommandOptions = { + startApp: true, + } + + @args.spread({ description: 'Group names, e.g. alt.binaries.hdtv' }) + declare names: string[] + + @flags.boolean({ description: 'Deactivate the groups instead' }) + declare disable: boolean + + @flags.boolean({ + description: 'Also download bodies to identify obfuscated posts (one body per article; bandwidth-heavy)', + }) + declare bodies: boolean + + @flags.number({ description: 'Start this many articles back instead of INDEX_INITIAL_BACKFILL (new groups only)' }) + declare backfill?: number + + @flags.boolean({ description: 'With --backfill: also for groups indexed before, skipping what was missed since' }) + declare reset: boolean + + public async run() { + for (const name of this.names) { + if (!/^[a-z0-9][a-z0-9.+_-]*$/i.test(name)) { + this.logger.error(`Not a valid group name: ${name}`) + continue + } + const group = await Group.firstOrCreate({ name }, { name, active: !this.disable, fetchBodies: this.bodies }) + group.active = !this.disable + if (this.bodies) group.fetchBodies = true + if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) { + // The scheduler continues after last_queued_id. + const last = await this.lastArticle(name) + group.lastQueuedId = (last > BigInt(this.backfill) ? last - BigInt(this.backfill) : 0n) + group.lastIndexedId = group.lastQueuedId + } + await group.save() + this.logger.info( + `${name}: ${group.active ? 'active' : 'inactive'}, bodies ${group.fetchBodies ? 'on' : 'off'}` + + (this.backfill !== undefined ? `, starting after article ${group.lastQueuedId}` : '') + ) + } + if (this.backfill !== undefined) await NntpService.shutdown() + } + + private async lastArticle(name: string): Promise { + const conn = await NntpService.acquire() + let failed = false + try { + return parseGroupResponse((await conn.group(name)).resp).last + } catch (error) { + failed = true + throw error + } finally { + NntpService.release(conn, failed) + } + } +} diff --git a/commands/GroupsTop.ts b/commands/GroupsTop.ts new file mode 100644 index 0000000..d848aac --- /dev/null +++ b/commands/GroupsTop.ts @@ -0,0 +1,176 @@ +import { writeFile } from 'node:fs/promises' +import { setTimeout as sleep } from 'node:timers/promises' +import { BaseCommand, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import NntpService from '#services/NntpService' +import indexerConfig from '#config/indexer' +import { parseGroupResponse } from '#lib/nntp/group' +import { parseSubject } from '#lib/subject/index' + +interface GroupRate { + name: string + perMinute: number + sampled?: number + normal?: number + obfuscated?: number + skipped?: number + avgArticleBytes?: number + avgHeaderBytes?: number +} + +/** + * Lists the busiest newsgroups (by articles posted per minute, from two + * LIST ACTIVE snapshots) and optionally samples their newest headers to + * estimate what indexing them would cost. See LOG.md (2026-09-28 (14)). + * + * node ace groups:top --minutes 5 --limit 1000 --sample 300 --out /data/top-groups.txt + */ +export default class GroupsTop extends BaseCommand { + public static commandName = 'groups:top' + public static description = 'Lists the busiest binary groups and estimates the bandwidth to index them.' + + public static options: CommandOptions = { + startApp: true, + } + + @flags.string({ description: 'Group pattern for LIST ACTIVE', default: 'alt.binaries.*' }) + declare pattern: string + + @flags.number({ description: 'Minutes between the two snapshots', default: 5 }) + declare minutes: number + + @flags.number({ description: 'Number of groups to list', default: 1000 }) + declare limit: number + + @flags.number({ description: 'Headers to sample per listed group (0 = no sampling)', default: 0 }) + declare sample: number + + @flags.string({ description: 'Write the list to this file (default: print only)' }) + declare out?: string + + public async run() { + const conn = await NntpService.acquire() + let failed = false + try { + this.logger.info(`Snapshot 1 of ${this.pattern}…`) + const first = await this.snapshot(conn) + this.logger.info(`${first.size} groups. Waiting ${this.minutes} min for snapshot 2…`) + const started = Date.now() + await sleep(this.minutes * 60_000) + const second = await this.snapshot(conn) + const minutes = (Date.now() - started) / 60_000 + + const groups: GroupRate[] = [] + for (const [name, high] of second) { + const before = first.get(name) + if (before === undefined || high <= before) continue + groups.push({ name, perMinute: Number(high - before) / minutes }) + } + groups.sort((a, b) => b.perMinute - a.perMinute) + const top = groups.slice(0, this.limit) + this.logger.info(`${groups.length} active groups; listing ${top.length}.`) + + if (this.sample > 0) { + for (const [i, group] of top.entries()) { + try { + Object.assign(group, await this.sampleGroup(conn, group.name)) + } catch (error: any) { + this.logger.warning(`${group.name}: ${error.message}`) + } + if ((i + 1) % 50 === 0) this.logger.info(`Sampled ${i + 1}/${top.length} groups…`) + } + } + + const text = this.format(top, minutes) + if (this.out) { + await writeFile(this.out, text) + this.logger.info(`Written to ${this.out}`) + } else { + console.log(text) + } + } catch (error) { + failed = true + throw error + } finally { + NntpService.release(conn, failed) + await NntpService.shutdown() + } + } + + /** Group name → highest article number, from LIST ACTIVE. */ + private async snapshot(conn: any): Promise> { + const { data } = await conn.list(this.pattern) + const result = new Map() + for (const line of Buffer.from(data).toString('latin1').split('\r\n')) { + const [name, high] = line.split(' ') + if (name && /^\d+$/.test(high ?? '')) result.set(name, BigInt(high)) + } + return result + } + + /** Classifies the newest headers the way worker:fetch does. */ + private async sampleGroup(conn: any, name: string) { + const { last } = parseGroupResponse((await conn.group(name)).resp) + const start = last - BigInt(this.sample) + 1n + const { overviews } = await conn.xover((start > 0n ? start : 1n).toString(), last.toString()) + let normal = 0 + let skipped = 0 + let obfuscated = 0 + let obfuscatedBytes = 0 + let headerBytes = 0 + for (const [, header] of overviews as [number, Record][]) { + if (!header) continue + headerBytes += Object.values(header).reduce((sum, v) => sum + String(v).length + 1, 0) + if (header.subject && parseSubject(header.subject)) normal++ + else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++ + else { + obfuscated++ + obfuscatedBytes += Number(header[':bytes']) || 0 + } + } + const sampled = normal + skipped + obfuscated + return { + sampled, + normal, + skipped, + obfuscated, + avgArticleBytes: obfuscated ? obfuscatedBytes / obfuscated : 0, + avgHeaderBytes: sampled ? headerBytes / sampled : 0, + } + } + + private format(groups: GroupRate[], minutes: number): string { + const sampled = groups.some((g) => g.sampled) + const lines = [ + `# Busiest ${this.pattern} groups by articles/min, measured over ${minutes.toFixed(1)} min on ${new Date().toISOString()}`, + sampled + ? '# rank\tgroup\tarticles/min\tobfuscated%\tskipped%\tbody Mbit/s\theader Mbit/s' + : '# rank\tgroup\tarticles/min', + ] + let bodyTotal = 0 + let headerTotal = 0 + let articlesTotal = 0 + for (const [i, g] of groups.entries()) { + articlesTotal += g.perMinute + if (!g.sampled) { + lines.push(`${i + 1}\t${g.name}\t${Math.round(g.perMinute)}`) + continue + } + const obfuscatedShare = g.obfuscated! / g.sampled + // Bodies: every obfuscated article is downloaded once. Headers: every article. + const bodyMbit = (g.perMinute * obfuscatedShare * g.avgArticleBytes! * 8) / 60 / 1e6 + const headerMbit = (g.perMinute * g.avgHeaderBytes! * 8) / 60 / 1e6 + bodyTotal += bodyMbit + headerTotal += headerMbit + lines.push( + [i + 1, g.name, Math.round(g.perMinute), (100 * obfuscatedShare).toFixed(1), ((100 * g.skipped!) / g.sampled).toFixed(1), bodyMbit.toFixed(1), headerMbit.toFixed(2)].join('\t') + ) + } + lines.push(`# total: ${Math.round(articlesTotal)} articles/min`) + if (sampled) { + lines.push(`# bandwidth to index like alt.binaries.test (headers + obfuscated bodies): ${(bodyTotal + headerTotal).toFixed(0)} Mbit/s`) + lines.push(`# of which bodies ${bodyTotal.toFixed(0)} Mbit/s, headers ${headerTotal.toFixed(1)} Mbit/s (skip list applied)`) + } + return lines.join('\n') + '\n' + } +} diff --git a/commands/IndexScheduler.ts b/commands/IndexScheduler.ts index 3367b44..2f33159 100644 --- a/commands/IndexScheduler.ts +++ b/commands/IndexScheduler.ts @@ -3,6 +3,10 @@ import { CommandOptions } from '@adonisjs/core/types/ace' import Group from '#models/group' import NntpService from '#services/NntpService' import QueueService from '#services/QueueService' +import indexerConfig from '#config/indexer' +import Shutdown from '#services/Shutdown' +import FetchRangeService from '#services/FetchRangeService' +import SplitService from '#services/SplitService' export default class IndexScheduler extends BaseCommand { public static commandName = 'index:scheduler' @@ -13,7 +17,6 @@ export default class IndexScheduler extends BaseCommand { } private pool = NntpService - private fetchQueue = QueueService.nntpFetchQueue public async run() { this.logger.info('Scheduler started. Awaiting tasks...') @@ -28,58 +31,89 @@ export default class IndexScheduler extends BaseCommand { } let conn + let failed = false try { conn = await this.pool.acquire() for (const group of groups) { try { - const groupInfo: any = await conn.group(group.name) - - // nntp-js returns article numbers as strings. We must parse them to BigInts. - const firstArticle = BigInt(groupInfo.first) - const lastArticle = BigInt(groupInfo.last) + const { first: firstArticle, last: lastArticle } = await this.pool.selectGroup( + conn, + group.name + ) - // lastIndexedId from the database should also be treated as a BigInt. - const lastIndexed = group.lastIndexedId ? BigInt(group.lastIndexedId) : null + await FetchRangeService.requeue(group) - const startId = lastIndexed ? lastIndexed + 1n : firstArticle + // Postgres returns bigint columns as strings. + const lastQueued = BigInt(group.lastQueuedId ?? 0) + const { batchSize, maxBatchesPerTick, initialBackfill } = indexerConfig + let startId = + lastQueued === 0n ? lastArticle - initialBackfill + 1n : lastQueued + 1n + if (startId < firstArticle) startId = firstArticle if (startId > lastArticle) { + await FetchRangeService.advanceWatermark(group) this.logger.info(`No new headers for group ${group.name}.`) continue } - const BATCH_SIZE = 100000n - const proposedEndId = startId + BATCH_SIZE - 1n - const endId = proposedEndId < lastArticle ? proposedEndId : lastArticle + let batches = 0 + let endId = startId - 1n + while (endId < lastArticle && batches < maxBatchesPerTick) { + const batchStart = endId + 1n + endId = batchStart + batchSize - 1n < lastArticle ? batchStart + batchSize - 1n : lastArticle + await FetchRangeService.queue(group, batchStart, endId) + batches++ + } - this.logger.info(`Queueing fetch job for ${group.name}: articles ${startId} to ${endId}`) - await this.fetchQueue.add('fetch-headers', { - groupName: group.name, - startId: startId.toString(), - endId: endId.toString(), - }) + this.logger.info( + `Queued ${batches} fetch job(s) for ${group.name}: articles ${startId} to ${endId}` + + (endId < lastArticle ? ` (${lastArticle - endId} left for next tick)` : '') + ) - group.lastIndexedId = endId + group.lastQueuedId = endId await group.save() + const watermark = await FetchRangeService.advanceWatermark(group) + if (endId - watermark > batchSize * BigInt(maxBatchesPerTick) * 3n) { + this.logger.warning( + `${group.name}: indexed up to ${watermark}, ${endId - watermark} articles behind the queue.` + ) + } } catch (err: any) { + failed = true this.logger.error(`Error processing group ${group.name}: ${err.message}`) } } } catch (err: any) { + failed = true this.logger.error(`Error in scheduler main loop: ${err.message}`) } finally { if (conn) { - this.pool.release(conn) + this.pool.release(conn, failed) } } } - // Run once immediately and then on an interval. - schedule() - setInterval(schedule, 60000) // 1 minute + const split = async () => { + try { + const result = await SplitService.splitPending() + if (result.parents > 0) { + this.logger.info( + `Split ${result.parents} conflicted binaries: ${result.volumes} verified volumes, ${result.completed} completed.` + ) + } + } catch (err: any) { + this.logger.error(`Error splitting conflicted binaries: ${err.message}`) + } + } - // Keep the command running - await new Promise(() => {}) + // Run immediately, then wait a full interval after each run so runs never overlap. + while (!Shutdown.requested) { + await schedule() + await split() + await Shutdown.sleep(indexerConfig.intervalMs) + } + await this.pool.shutdown() + await QueueService.closeAll() } } diff --git a/commands/ReleaseRebuild.ts b/commands/ReleaseRebuild.ts new file mode 100644 index 0000000..d8305c2 --- /dev/null +++ b/commands/ReleaseRebuild.ts @@ -0,0 +1,163 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import db from '@adonisjs/lucid/services/db' +import IngestService from '#services/IngestService' +import NameService from '#services/NameService' +import QueueService from '#services/QueueService' +import { rarReleaseIdentity, releaseIdentity } from '#lib/release/index' + +const BATCH = 5000 + +/** + * Recomputes the release of every binary (and its file) with the current + * rules in lib/release. Used after changing those rules. + */ +export default class ReleaseRebuild extends BaseCommand { + public static commandName = 'release:rebuild' + public static description = 'Reassigns all binaries and files to releases using the current grouping rules.' + + public static options: CommandOptions = { + startApp: true, + } + + @flags.boolean({ + description: + 'Also drop everything derived from file contents (par2 lists, par2/RAR groupings, real names), ' + + 'to rebuild it with content:backfill --all', + }) + declare resetContent: boolean + + public async run() { + if (this.resetContent) await this.dropContentDerived() + + let lastId = 0 + let assigned = 0 + let unassigned = 0 + try { + while (true) { + const binaries = await db + .from('binaries') + .where('id', '>', lastId) + .orderBy('id') + .limit(BATCH) + .select('id', 'poster', 'subject', 'name', 'groups', 'posted_at') + if (binaries.length === 0) break + lastId = Number(binaries.at(-1).id) + + const segments = binaries.map((binary) => { + // Body-derived binaries store a synthetic subject, see segmentFromYenc. + const weak = /^"[^"]*" yEnc$/.test(binary.subject) + return { + id: Number(binary.id), + poster: binary.poster, + groups: binary.groups, + postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined, + weakIdentity: weak, + release: releaseIdentity({ poster: binary.poster, subject: binary.subject, name: binary.name, weak }), + } + }) + + await db.transaction(async (trx) => { + const releaseIds = await IngestService.upsertReleases(trx.knexClient, segments) + const ids = segments.map((s) => s.id) + const releaseIdsPerBinary = segments.map((s) => + s.release ? releaseIds.get(s.release.key)! : null + ) + await trx.rawQuery( + `UPDATE binaries b SET release_id = v.release_id + FROM unnest(?::bigint[], ?::bigint[]) AS v(id, release_id) + WHERE b.id = v.id AND b.release_id IS DISTINCT FROM v.release_id + AND NOT (v.release_id IS NULL + AND b.release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar'))) + -- A posted NZB decided this grouping (NzbImportService); name rules don't override it. + AND NOT EXISTS (SELECT 1 FROM files n WHERE n.release_id = b.release_id AND n.content_type = 'nzb')`, + [ids, releaseIdsPerBinary] as any + ) + }) + assigned += segments.filter((s) => s.release).length + unassigned += segments.filter((s) => !s.release).length + } + + // Files grouped by their RAR headers: regroup from the stored listing (no downloads). + let rarGrouped = 0 + let rarCursor = 0 + while (true) { + const rows = await db + .from('files as f') + .join('binaries as b', 'b.file_id', 'f.id') + .whereNotNull('f.archive') + .whereNull('b.release_id') + .where('f.id', '>', rarCursor) + .orderBy('f.id') + .limit(BATCH) + .select('f.id as file_id', 'f.archive', 'b.id as binary_id', 'b.poster', 'b.groups', 'b.posted_at') + if (rows.length === 0) break + rarCursor = Number(rows.at(-1).file_id) + const segments = rows.map((r) => ({ + fileId: Number(r.file_id), + binaryId: Number(r.binary_id), + poster: r.poster, + groups: r.groups, + postedAt: r.posted_at ? new Date(r.posted_at) : undefined, + weakIdentity: true, + release: rarReleaseIdentity(r.archive?.entries ?? []), + })) + const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), segments) + for (const s of segments.filter((x) => x.release)) { + const releaseId = ids.get(s.release!.key) + await db.from('binaries').where('id', s.binaryId).update({ release_id: releaseId }) + await db.from('files').where('id', s.fileId).update({ release_id: releaseId }) + rarGrouped++ + } + } + + // Real names from par2 for binaries with a known size (no downloads). + let named = 0 + let nameCursor = 0 + while (true) { + const rows = await db.from('binaries').whereNotNull('size').where('id', '>', nameCursor).orderBy('id').limit(BATCH).select('id') + if (rows.length === 0) break + nameCursor = Number(rows.at(-1).id) + named += await NameService.resolve(rows.map((r) => Number(r.id))) + } + + const files = await db.rawQuery( + `UPDATE files f SET release_id = b.release_id FROM binaries b + WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id` + ) + const orphans = await db.rawQuery( + `DELETE FROM releases r WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id) + AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)` + ) + if (this.resetContent) { + this.logger.info('Content-derived data dropped; run `node ace content:backfill --all` to rebuild it.') + } + this.logger.info( + `Binaries: ${assigned} in a release, ${unassigned} without, ${rarGrouped} regrouped from RAR headers, ` + + `${named} named from par2. ` + + `Files updated: ${files.rowCount}. Empty releases removed: ${orphans.rowCount}.` + ) + } finally { + await QueueService.closeAll() + } + } + + /** par2 lists, par2/RAR release assignments and recovered names. */ + private async dropContentDerived() { + await db.transaction(async (trx) => { + await trx.rawQuery('DELETE FROM release_files') + await trx.rawQuery(`UPDATE binaries SET real_name = NULL WHERE real_name IS NOT NULL`) + await trx.rawQuery(`UPDATE files SET real_name = NULL, archive = NULL WHERE real_name IS NOT NULL OR archive IS NOT NULL`) + await trx.rawQuery(`UPDATE releases SET contents = NULL, encrypted = NULL WHERE contents IS NOT NULL OR encrypted IS NOT NULL`) + // Binaries whose name rules find no release were grouped from contents (or wrongly by size). + await trx.rawQuery(` + UPDATE binaries SET release_id = NULL + WHERE subject ~ '^"[^"]*" yEnc$' AND release_id IS NOT NULL + AND (name !~ '\\.[A-Za-z0-9]{1,5}$' OR release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))`) + await trx.rawQuery(` + UPDATE files f SET release_id = b.release_id FROM binaries b + WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id`) + }) + this.logger.info('Dropped par2 lists, content-derived groupings and recovered names.') + } +} diff --git a/commands/ReleaseRename.ts b/commands/ReleaseRename.ts new file mode 100644 index 0000000..f06f25d --- /dev/null +++ b/commands/ReleaseRename.ts @@ -0,0 +1,20 @@ +import { BaseCommand } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import { renameFromStoredContents } from '#services/ArchiveService' +import { regroupAndRenameFromPar2 } from '#services/NameService' + +/** Renames random-named releases from their stored archive listings (no downloads). */ +export default class ReleaseRename extends BaseCommand { + public static commandName = 'release:rename' + public static description = 'Renames random-named releases from their par2 listings and archive contents.' + + public static options: CommandOptions = { + startApp: true, + } + + public async run() { + const par2 = await regroupAndRenameFromPar2() + this.logger.info(`par2: ${par2.regrouped} files moved into the set that names them, ${par2.renamed} releases renamed.`) + this.logger.info(`Renamed ${await renameFromStoredContents()} releases from their archive contents.`) + } +} diff --git a/commands/ReleaseStats.ts b/commands/ReleaseStats.ts new file mode 100644 index 0000000..53c0704 --- /dev/null +++ b/commands/ReleaseStats.ts @@ -0,0 +1,57 @@ +import { BaseCommand } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import db from '@adonisjs/lucid/services/db' + +/** Prints how well files are being grouped into releases. */ +export default class ReleaseStats extends BaseCommand { + public static commandName = 'release:stats' + public static description = 'Shows release recognition statistics.' + + public static options: CommandOptions = { + startApp: true, + } + + public async run() { + const { rows } = await db.rawQuery(` + WITH r AS ( + SELECT r.id, r.file_total, r.poster IS NULL AS weak, r.source, + count(b.id) AS binaries, + count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed + FROM releases r LEFT JOIN binaries b ON b.release_id = r.id + GROUP BY r.id + ) + SELECT + count(*) AS releases, + count(*) FILTER (WHERE completed > 0) AS with_files, + count(*) FILTER (WHERE completed >= 2) AS multi_file, + count(*) FILTER (WHERE file_total IS NOT NULL AND completed >= file_total) AS complete_by_counter, + count(*) FILTER (WHERE weak) AS from_obfuscated, + count(*) FILTER (WHERE weak AND completed > 0) AS from_obfuscated_with_files + FROM r`) + const { rows: coverage } = await db.rawQuery(` + SELECT count(*) AS completed_files, + count(*) FILTER (WHERE release_id IS NOT NULL) AS in_release + FROM binaries WHERE completed_at IS NOT NULL`) + const { rows: bySource } = await db.rawQuery(` + SELECT coalesce(r.source, 'unknown') AS source, count(DISTINCT r.id) AS releases, + count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed_files + FROM releases r JOIN binaries b ON b.release_id = r.id GROUP BY 1 ORDER BY 2 DESC`) + // Readable names: descriptive as posted, or recovered from contents. + const { rows: naming } = await db.rawQuery(` + WITH f AS ( + SELECT real_name IS NOT NULL AS recovered, + is_readable_name(coalesce(real_name, filename)) AS readable_name, + regexp_replace(coalesce(real_name, filename), + '(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|mkv|mp4|avi|flac|mp3|jpg))+$', '', 'i') AS stem, + coalesce(real_name, filename) AS name + FROM files + ) + SELECT count(*) AS files, + count(*) FILTER (WHERE recovered) AS names_recovered, + count(*) FILTER (WHERE name ~ '\\.[A-Za-z0-9]{1,5}$' AND readable_name) AS readable + FROM f`) + this.logger.info(JSON.stringify({ ...rows[0], ...coverage[0] })) + this.logger.info(`naming: ${JSON.stringify(naming[0])}`) + this.logger.info(`by source: ${JSON.stringify(bySource)}`) + } +} diff --git a/commands/workers/Body.ts b/commands/workers/Body.ts index a05815b..1886eec 100644 --- a/commands/workers/Body.ts +++ b/commands/workers/Body.ts @@ -4,9 +4,13 @@ import { Worker } from 'bullmq' import fs from 'node:fs/promises' import path from 'node:path' import queueConfig from '#config/queue' +import Shutdown from '#services/Shutdown' import QueueService from '#services/QueueService' +import env from '#start/env' import NntpService from '#services/NntpService' -import { parseYencMeta } from '#services/YencService' +import IngestService, { segmentFromYenc } from '#services/IngestService' +import { parseYencMeta, YencError } from '#lib/yenc/index' +import { firstChunkMd5 } from '#services/ContentService' export default class BodyWorker extends BaseCommand { public static commandName = 'worker:body' @@ -20,31 +24,26 @@ export default class BodyWorker extends BaseCommand { this.logger.info('Starting body worker...') const pool = NntpService - const headerQueue = QueueService.headerQueue const worker = new Worker('body-queue', async (job) => { - const { header, group } = job.data + const { header, group, articleNumber } = job.data const messageId = header['message-id'] this.logger.debug(`Processing header with unparsable subject: ${header.subject}`) let conn + let failed = false try { conn = await pool.acquire() - const bodyBuffer: Buffer = (await conn.body(messageId)).data + const bodyBuffer = await pool.fetchBody(conn, messageId) + let meta try { - const meta = parseYencMeta(bodyBuffer) - if (meta.header.name) { - const { name, part, total } = meta.header - const newSubject = `"${name}" yEnc (${part}/${total})` - header.subject = newSubject - - this.logger.info(`Found yEnc metadata in body. New subject: ${newSubject}`) - await headerQueue.add('process-header', { header, group }) - } else { - this.logger.warning(`Could not find yEnc metadata in body for header: ${header.subject}`) - } + meta = parseYencMeta(bodyBuffer) } catch (parseError: any) { + if (parseError instanceof YencError && bodyBuffer.indexOf('=ybegin ') === -1) { + this.logger.debug(`Not a yEnc post, skipping: ${messageId}`) + return + } this.logger.error(`Failed to parse yEnc data for message ID ${messageId}. Dumping buffer.`) const debugDir = path.join(this.app.appRoot.pathname, 'debug') await fs.mkdir(debugDir, { recursive: true }) @@ -54,21 +53,45 @@ export default class BodyWorker extends BaseCommand { this.logger.error(`Problematic body buffer saved to: ${dumpFile}`) throw parseError } + + pool.release(conn) + conn = undefined + + const segment = segmentFromYenc(header, group, meta, articleNumber) + if (segment?.number === 1) segment.md5_16k = firstChunkMd5(bodyBuffer, meta.begin.size) + if (!segment) { + this.logger.warning(`Inconsistent yEnc part numbering in ${messageId}, skipping.`) + return + } + const completed = await IngestService.ingest([segment]) + this.logger.debug( + `Stored part ${segment.number}/${segment.totalParts} of "${segment.name}"` + + (completed.length > 0 ? ' (file complete)' : '') + ) } catch (error: any) { + failed = true this.logger.error(`Error in body worker for message ID ${messageId}: ${error.message}`) throw error } finally { if (conn) { - pool.release(conn) + pool.release(conn, failed) } } - }, { connection: queueConfig.connection }) + }, { + connection: queueConfig.connection, + // Each job holds one NNTP connection while downloading, so keep this <= the pool size. + concurrency: env.get('BODY_CONCURRENCY', 1), + }) worker.on('failed', (job, err) => { this.logger.error(`Body job ${job?.id} failed: ${err.message}`) }) this.logger.info('Body worker started and listening for jobs.') - await new Promise(() => {}) + await Shutdown.wait() + this.logger.info('Shutting down: finishing active jobs...') + await worker.close() + await NntpService.shutdown() + await QueueService.closeAll() } } diff --git a/commands/workers/Collection.ts b/commands/workers/Collection.ts index 9796181..c4b68e9 100644 --- a/commands/workers/Collection.ts +++ b/commands/workers/Collection.ts @@ -2,10 +2,12 @@ import { BaseCommand } from '@adonisjs/core/ace' import { CommandOptions } from '@adonisjs/core/types/ace' import { Worker } from 'bullmq' import queueConfig from '#config/queue' +import env from '#start/env' +import Shutdown from '#services/Shutdown' +import QueueService from '#services/QueueService' import NntpService from '#services/NntpService' -import { YencFile } from '#services/YencFile' +import ContentService from '#services/ContentService' import File from '#models/file' -import { createExtractorFromData } from 'node-unrar-js' export default class CollectionWorker extends BaseCommand { public static commandName = 'worker:collection' @@ -18,8 +20,6 @@ export default class CollectionWorker extends BaseCommand { public async run() { this.logger.info('Starting collection worker...') - const pool = NntpService - const worker = new Worker('collection-queue', async (job) => { const { fileId } = job.data this.logger.debug(`Processing file ID ${fileId} for collection.`) @@ -30,51 +30,47 @@ export default class CollectionWorker extends BaseCommand { return } - const RAR_REGEX = /\.part0*1\.rar$/ - if (RAR_REGEX.test(file.filename)) { - this.logger.info(`File "${file.filename}" is the first part of a RAR set.`) - - const firstPart = file.messageIds['1'] - if (!firstPart || !firstPart.id) { - this.logger.error(`Could not find message ID for the first part of file "${file.filename}".`) - return + try { + const content = await ContentService.inspect(fileId) + if (content?.par2) { + const { par2 } = content + this.logger.info(`Par2 "${file.filename}": release "${par2.releaseName}", ${par2.listed} files listed.`) + } else if (content?.archive) { + const { info, releaseRenamed, grouped } = content.archive + const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files'] + .filter(Boolean) + .join(', ') + this.logger.info( + `Archive "${file.filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` + + (grouped ? ', grouped into a release' : '') + + (releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '') + ) } - - let conn - try { - conn = await pool.acquire() - const bodyBuffer = (await conn.body(`<${firstPart.id}>`)).data - - const yencFile = new YencFile() - yencFile.processPart(bodyBuffer) - const decodedBuffer = yencFile.getBuffer() - - const extractor = await createExtractorFromData({ data: new Uint8Array(decodedBuffer).buffer }) - const fileList = extractor.getFileList() - // In a real implementation, we would save this file list. - this.logger.info(`Files in "${file.filename}": ${JSON.stringify(fileList)}`) - - } catch (error: any) { - if (error.code === 430) { - this.logger.error(`Article not found for first part of RAR set (Message ID: ${firstPart.id})`) - } else { - this.logger.error(`Error processing RAR file: ${error.message}`) - } - } finally { - if (conn) { - pool.release(conn) - } + if (content?.named) this.logger.info(`"${file.filename}" got its real name from par2.`) + if (content?.contentType === 'nzbe') this.logger.info(`"${file.filename}" is an encrypted NZB container.`) + if (content?.contentType === 'rclone') this.logger.info(`"${file.filename}" is rclone-encrypted.`) + if (content?.nzb) { + const n = content.nzb + this.logger.info(`NZB "${file.filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`) } - } else { - this.logger.debug(`File "${file.filename}" is not the first part of a RAR set.`) + } catch (error: any) { + this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`) } - }, { connection: queueConfig.connection }) + }, { + connection: queueConfig.connection, + // Each job holds one NNTP connection while downloading; keep <= the pool size. + concurrency: env.get('COLLECTION_CONCURRENCY', 1), + }) worker.on('failed', (job, err) => { this.logger.error(`Collection job ${job?.id} failed: ${err.message}`) }) this.logger.info('Collection worker started and listening for jobs.') - await new Promise(() => {}) + await Shutdown.wait() + this.logger.info('Shutting down: finishing active jobs...') + await worker.close() + await NntpService.shutdown() + await QueueService.closeAll() } } diff --git a/commands/workers/Deferred.ts b/commands/workers/Deferred.ts new file mode 100644 index 0000000..008c2ca --- /dev/null +++ b/commands/workers/Deferred.ts @@ -0,0 +1,57 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import env from '#start/env' +import Shutdown from '#services/Shutdown' +import NntpService from '#services/NntpService' +import QueueService from '#services/QueueService' +import DeferredService from '#services/DeferredService' + +const IDLE_MS = 60_000 + +/** + * Works through `deferred_files` (CONCEPT.md D28): one body per file, and + * small files completely. Not started by default (compose profile + * `deferred`); run it when there is bandwidth to spare. + */ +export default class DeferredWorker extends BaseCommand { + public static commandName = 'worker:deferred' + public static description = 'Samples deferred files (e.g. ngPost uploads): one article body per file.' + + public static options: CommandOptions = { + startApp: true, + } + + @flags.number({ description: 'Stop after this many files (default: keep running)' }) + declare limit?: number + + public async run() { + const concurrency = env.get('DEFERRED_CONCURRENCY', 4) + this.logger.info(`Deferred worker started (concurrency ${concurrency}).`) + const counts: Record = {} + let done = 0 + try { + while (!Shutdown.requested && (!this.limit || done < this.limit)) { + const batch = await DeferredService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity)) + if (batch.length === 0) { + if (this.limit) break + await Shutdown.sleep(IDLE_MS) + continue + } + let next = 0 + await Promise.all( + Array.from({ length: concurrency }, async () => { + while (next < batch.length) { + const status = await DeferredService.process(batch[next++]) + counts[status] = (counts[status] ?? 0) + 1 + } + }) + ) + done += batch.length + this.logger.info(`${done} deferred files processed: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`) + } + } finally { + await NntpService.shutdown() + await QueueService.closeAll() + } + } +} diff --git a/commands/workers/Fetch.ts b/commands/workers/Fetch.ts index c92b038..f970e01 100644 --- a/commands/workers/Fetch.ts +++ b/commands/workers/Fetch.ts @@ -2,12 +2,20 @@ import { BaseCommand } from '@adonisjs/core/ace' import { CommandOptions } from '@adonisjs/core/types/ace' import { Worker } from 'bullmq' import queueConfig from '#config/queue' +import env from '#start/env' +import db from '@adonisjs/lucid/services/db' +import indexerConfig from '#config/indexer' +import Shutdown from '#services/Shutdown' import QueueService from '#services/QueueService' import NntpService from '#services/NntpService' +import FetchRangeService from '#services/FetchRangeService' +import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService' +import DeferredService, { bodyJob, isDeferred, type DeferredArticle } from '#services/DeferredService' +import SpotService, { isSpotGroup } from '#services/SpotService' export default class FetchWorker extends BaseCommand { public static commandName = 'worker:fetch' - public static description = 'Starts a worker to fetch headers from the NNTP server.' + public static description = 'Starts a worker that fetches headers (XOVER) and stores them as binaries/parts.' public static options: CommandOptions = { startApp: true, @@ -17,53 +25,86 @@ export default class FetchWorker extends BaseCommand { this.logger.info('Starting fetch worker...') const pool = NntpService - const headerQueue = QueueService.headerQueue + const bodyQueue = QueueService.bodyQueue const worker = new Worker('nntp-fetch-queue', async (job) => { const { groupName, startId, endId } = job.data this.logger.info(`Processing fetch job for ${groupName}, articles ${startId}-${endId}`) + let overviews: [number, Record][] let conn + let failed = false try { conn = await pool.acquire() await conn.group(groupName) - const overview: any = await conn.xover(startId, endId) - this.logger.info(`Fetched ${overview.overviews.length} headers from ${groupName}.`) - - if (overview.overviews.length > 0) { - const jobs = overview.overviews.map(([id, header]: [number, any]) => { - if (!header) { - this.logger.warning(`Header is undefined for job ${job.id}`) - return null - } - return { - name: 'process-header', - data: { header, group: groupName }, - opts: { jobId: `${groupName}-${id}` }, - } - }).filter(Boolean) - - await headerQueue.addBulk(jobs) - this.logger.info(`Added ${jobs.length} header jobs to the queue for ${groupName}.`) - } + overviews = (await conn.xover(startId, endId)).overviews } catch (error: any) { + failed = true this.logger.error(`Error fetching headers for ${groupName}: ${error.message}`) throw error } finally { if (conn) { - pool.release(conn) + pool.release(conn, failed) } } - }, { + + if (isSpotGroup(groupName)) { + const spots = await SpotService.recordHeaders(groupName, overviews) + if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId) + this.logger.info(`${groupName} ${startId}-${endId}: ${overviews.length} headers, ${spots} spots queued.`) + return + } + + const group = await db.from('groups').where('name', groupName).first() + const fetchBodies = Boolean(group?.fetch_bodies) + + const segments: IngestSegment[] = [] + const obfuscated: { articleNumber: number; header: Record }[] = [] + const deferred: DeferredArticle[] = [] + let skipped = 0 + for (const [articleNumber, header] of overviews) { + if (!header) continue + const segment = segmentFromHeader(header, groupName, articleNumber) + if (segment) segments.push(segment) + // Recorded per file (or as article ranges) in every group, bodies or not; processed later if at all. + else if (isDeferred(header)) deferred.push({ articleNumber, header }) + else if (!fetchBodies || indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++ + else obfuscated.push({ articleNumber, header }) + } + + const completed = await IngestService.ingest(segments) + const deferredFiles = await DeferredService.record(groupName, deferred) + + if (obfuscated.length > 0) { + await bodyQueue.addBulk(obfuscated.map(({ articleNumber, header }) => bodyJob(header, groupName, articleNumber))) + } + + if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId) + + this.logger.info( + `${groupName} ${startId}-${endId}: ${overviews.length} headers, ${segments.length} stored, ` + + `${obfuscated.length} sent to body-queue${skipped ? ` (${skipped} skipped)` : ''}` + + `${deferred.length ? `, ${deferred.length} deferred (${deferredFiles} files)` : ''}, ${completed.length} files completed.` + ) + }, { connection: queueConfig.connection, - concurrency: 5, + // XOVER is quick; most of a job is storing. Each concurrent job uses one DB connection (DB_POOL_MAX). + concurrency: env.get('FETCH_CONCURRENCY', 5), }) - worker.on('failed', (job, err) => { + worker.on('failed', async (job, err) => { this.logger.error(`Fetch job ${job?.id} failed for group ${job?.data.groupName}: ${err.message}`) + // Only after the last attempt; the scheduler re-queues the range later. + if (job?.data.rangeId && job.attemptsMade >= (job.opts.attempts ?? 1)) { + await FetchRangeService.markFailed(job.data.rangeId, err.message).catch(() => {}) + } }) this.logger.info('Fetch worker started and listening for jobs.') - await new Promise(() => {}) // Keep command running + await Shutdown.wait() + this.logger.info('Shutting down: finishing active jobs...') + await worker.close() + await NntpService.shutdown() + await QueueService.closeAll() } } diff --git a/commands/workers/File.ts b/commands/workers/File.ts index b498dfa..cea49dc 100644 --- a/commands/workers/File.ts +++ b/commands/workers/File.ts @@ -1,13 +1,14 @@ import { BaseCommand } from '@adonisjs/core/ace' import { CommandOptions } from '@adonisjs/core/types/ace' import { Worker } from 'bullmq' +import db from '@adonisjs/lucid/services/db' import queueConfig from '#config/queue' +import Shutdown from '#services/Shutdown' import QueueService from '#services/QueueService' -import File from '#models/file' export default class FileWorker extends BaseCommand { public static commandName = 'worker:file' - public static description = 'Starts a worker to process completed files.' + public static description = 'Starts a worker that turns completed binaries into files.' public static options: CommandOptions = { startApp: true, @@ -19,45 +20,78 @@ export default class FileWorker extends BaseCommand { const collectionQueue = QueueService.collectionQueue const worker = new Worker('file-queue', async (job) => { - const { filename, parts, groups } = job.data - const partCount = Object.keys(parts).length - this.logger.debug(`Processing complete file: "${filename}" with ${partCount} parts.`) + const { binaryId } = job.data - const firstPart = JSON.parse(Object.values(parts)[0] as string) - const poster = firstPart.from - const date = new Date(firstPart.date).getTime() - - const messageIds = Object.entries(parts).reduce((acc, [partNumber, partData]) => { - const part = JSON.parse(partData as string) - const messageId = part['message-id'] - if (messageId) { - acc[partNumber] = { - id: messageId.replace(/[<>]/g, ''), - size: part[':bytes'], - } - } else { - this.logger.warning(`Message ID not found for part ${partNumber} of file "${filename}"`) - } - return acc - }, {} as Record) - - if (Object.keys(messageIds).length !== partCount) { - throw new Error(`Could not process all parts for file "${filename}" due to missing message IDs.`) + const binary = await db.from('binaries').where('id', binaryId).first() + if (!binary) { + this.logger.warning(`Binary ${binaryId} no longer exists (garbage collected?).`) + return } - const file = await File.create({ - filename, - poster, - date, - parts: partCount, - messageIds, - groups, - }) + if (binary.conflicted_at) { + this.logger.warning(`Binary ${binaryId} ("${binary.name}") has conflicting parts, skipping.`) + return + } - this.logger.debug(`Saved file "${filename}" to database with ID: ${file.id}`) + const parts = await db + .from('parts') + .where('binary_id', binaryId) + .orderBy('number') + .select('number', 'message_id', 'bytes') + if (parts.length < binary.total_parts) { + throw new Error(`Binary ${binaryId} has ${parts.length}/${binary.total_parts} parts.`) + } - await collectionQueue.add('process-collection', { fileId: file.id }) - this.logger.debug(`Added file ID ${file.id} to collection queue.`) + const messageIds = Object.fromEntries( + parts.map((part) => [part.number, { id: part.message_id, size: part.bytes }]) + ) + + // Already known as a file, e.g. added from a posted NZB before its own articles completed. + const existing = await db + .from('files') + .whereRaw(`(message_ids->'1'->>'id') = ?`, [messageIds['1']?.id ?? '']) + .first() + if (existing) { + await db.from('binaries').where('id', binaryId).update({ file_id: existing.id }) + await dropCopiedParts(binary) + this.logger.debug(`File "${binary.name}" already exists as ID ${existing.id} (same first segment).`) + return + } + + // file_key is unique: a second completion of the same upload inserts nothing. + const knex = db.connection().getWriteClient() + const now = new Date() + const [inserted] = await knex('files') + .insert({ + file_key: binary.file_key, + filename: binary.name, + poster: binary.poster, + date: binary.posted_at ? new Date(binary.posted_at).getTime() : now.getTime(), + parts: parts.length, + message_ids: JSON.stringify(messageIds), + groups: JSON.stringify(binary.groups), + release_id: binary.release_id, + real_name: binary.real_name, + size: parts.reduce((sum, part) => sum + Number(part.bytes), 0), + created_at: now, + updated_at: now, + }) + .onConflict('file_key') + .ignore() + .returning('id') + + const fileId = + inserted?.id ?? (await db.from('files').where('file_key', binary.file_key).first())?.id + await db.from('binaries').where('id', binaryId).update({ file_id: fileId }) + await dropCopiedParts(binary) + + if (!inserted) { + this.logger.debug(`File "${binary.name}" (${binary.file_key}) already exists as ID ${fileId}.`) + return + } + + this.logger.info(`Saved file "${binary.name}" (${parts.length} parts) as ID ${fileId}.`) + await collectionQueue.add('process-collection', { fileId }, { jobId: `file-${fileId}` }) }, { connection: queueConfig.connection }) worker.on('failed', (job, err) => { @@ -65,6 +99,19 @@ export default class FileWorker extends BaseCommand { }) this.logger.info('File worker started and listening for jobs.') - await new Promise(() => {}) + await Shutdown.wait() + this.logger.info('Shutting down: finishing active jobs...') + await worker.close() + await QueueService.closeAll() } } + +/** + * The file holds every message id now. Binaries identified by their subject + * (no size: header-derived) don't need their parts any more; body-derived + * ones keep them, so articles of an equal-sized next volume are recognised + * as a conflict and split off (CONCEPT.md D20). LOG.md 2026-09-28 (17). + */ +async function dropCopiedParts(binary: { id: number; size: number | null }) { + if (binary.size === null) await db.from('parts').where('binary_id', binary.id).delete() +} diff --git a/commands/workers/Header.ts b/commands/workers/Header.ts deleted file mode 100644 index e53997a..0000000 --- a/commands/workers/Header.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { BaseCommand } from '@adonisjs/core/ace' -import { CommandOptions } from '@adonisjs/core/types/ace' -import { Worker } from 'bullmq' -import queueConfig from '#config/queue' -import QueueService from '#services/QueueService' -import RedisService from '#services/RedisService' - -export default class HeaderWorker extends BaseCommand { - public static commandName = 'worker:header' - public static description = 'Starts a worker to process headers.' - - public static options: CommandOptions = { - startApp: true, - } - - public async run() { - this.logger.info('Starting header worker...') - - const redis = RedisService.client - const fileQueue = QueueService.fileQueue - const bodyQueue = QueueService.bodyQueue - - const worker = new Worker('header-queue', async (job) => { - const { header, group } = job.data - if (!header || !header.subject) { - this.logger.warning(`Received job with invalid header data. JobID: ${job.id}`) - return - } - - const subject = header.subject - const SUBJECT_REGEX = /"(.+)"(?: yEnc)? \((\d+)\/(\d+)\)/ - const match = subject.match(SUBJECT_REGEX) - - if (match) { - const filename = match[1] - const part = parseInt(match[2], 10) - const total = parseInt(match[3], 10) - - const fileKey = `file:${filename}` - await redis.hset(fileKey, part, JSON.stringify(header)) - - const partCount = await redis.hlen(fileKey) - - if (partCount === total) { - const fileParts = await redis.hgetall(fileKey) - await fileQueue.add('process-file', { filename, parts: fileParts, groups: [group] }) - await redis.del(fileKey) - this.logger.info(`File "${filename}" is complete and moved to file-queue.`) - } else { - this.logger.info(`Stored part ${part}/${total} for file "${filename}"`) - } - } else { - this.logger.warning(`Could not parse subject: "${subject}". Moving to body-queue.`) - await bodyQueue.add('process-body', { header, group }) - } - }, { connection: queueConfig.connection }) - - worker.on('failed', (job, err) => { - this.logger.error(`Header job ${job?.id} failed: ${err.message}`) - }) - - this.logger.info('Header worker started and listening for jobs.') - await new Promise(() => {}) - } -} diff --git a/commands/workers/Spots.ts b/commands/workers/Spots.ts new file mode 100644 index 0000000..ae47a20 --- /dev/null +++ b/commands/workers/Spots.ts @@ -0,0 +1,54 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import { CommandOptions } from '@adonisjs/core/types/ace' +import env from '#start/env' +import Shutdown from '#services/Shutdown' +import NntpService from '#services/NntpService' +import SpotService from '#services/SpotService' + +const IDLE_MS = 30_000 + +/** + * Works through queued Spotnet spots (CONCEPT.md D29): verifies each one, + * downloads its compressed NZB (a few KB to a few MB) and imports it. + */ +export default class SpotsWorker extends BaseCommand { + public static commandName = 'worker:spots' + public static description = 'Reads Spotnet spots: verifies them and imports their NZBs.' + + public static options: CommandOptions = { + startApp: true, + } + + @flags.number({ description: 'Stop after this many spots (default: keep running)' }) + declare limit?: number + + public async run() { + const concurrency = env.get('SPOT_CONCURRENCY', 4) + this.logger.info(`Spot worker started (concurrency ${concurrency}).`) + const counts: Record = {} + let done = 0 + try { + while (!Shutdown.requested && (!this.limit || done < this.limit)) { + const batch = await SpotService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity)) + if (batch.length === 0) { + if (this.limit) break + await Shutdown.sleep(IDLE_MS) + continue + } + let next = 0 + await Promise.all( + Array.from({ length: concurrency }, async () => { + while (next < batch.length) { + const status = await SpotService.process(batch[next++]) + counts[status] = (counts[status] ?? 0) + 1 + } + }) + ) + done += batch.length + this.logger.info(`${done} spots: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`) + } + } finally { + await NntpService.shutdown() + } + } +} diff --git a/config/database.ts b/config/database.ts index f50be91..42d262e 100644 --- a/config/database.ts +++ b/config/database.ts @@ -61,7 +61,13 @@ const dbConfig = defineConfig({ user: env.get('DB_USER'), password: env.get('DB_PASSWORD')?.release(), database: env.get('DB_DATABASE'), + // For idempotent bulk writers (worker:fetch): don't wait for the WAL + // flush on every statement. A crash loses the last moments of writes, + // which the retried job writes again. + ...(env.get('DB_SYNCHRONOUS_COMMIT') === 'off' ? { options: '-c synchronous_commit=off' } : {}), }, + // One connection per concurrent job; knex defaults to 10. + pool: { min: 0, max: env.get('DB_POOL_MAX', 10) }, migrations: { naturalSort: true, paths: ['database/migrations'], diff --git a/config/gc.ts b/config/gc.ts new file mode 100644 index 0000000..ff144b4 --- /dev/null +++ b/config/gc.ts @@ -0,0 +1,57 @@ +import env from '#start/env' + +/** + * Retention for transient pipeline state in Redis. See CONCEPT.md (D11). + */ +export default { + /** How often `gc:scheduler` sweeps. */ + intervalMs: env.get('GC_INTERVAL_MS', 10 * 60 * 1000), + + jobs: { + /** Completed jobs are only kept for inspection; keep them briefly. */ + completedAgeSeconds: env.get('GC_COMPLETED_AGE_SECONDS', 60 * 60), + completedMaxCount: 1000, + /** Failed jobs are kept longer so they can be inspected and retried. */ + failedAgeSeconds: env.get('GC_FAILED_AGE_SECONDS', 7 * 24 * 60 * 60), + }, + + /** + * Upper bound on waiting body jobs. Obfuscated traffic can exceed what the + * line downloads for hours; beyond this, the oldest waiting jobs are + * dropped so Redis stays bounded (~1 KB per job). + */ + bodyQueueMax: env.get('GC_BODY_QUEUE_MAX', 500_000), + + binaries: { + /** + * A binary that gets no new segments for this long is deleted: incomplete + * ones are given up on, completed ones have been copied into `files`. + */ + staleAfterSeconds: env.get('GC_BINARY_STALE_SECONDS', 48 * 60 * 60), + /** + * Header-derived binaries that still have one article after this long: + * uploads that give every article its own random name (and poster) never + * complete from headers. A real file gets its next articles in minutes. + */ + singleArticleAfterSeconds: env.get('GC_SINGLE_ARTICLE_SECONDS', 2 * 60 * 60), + }, + + /** Releases without files or binaries (their binaries were collected) are deleted after this long. */ + emptyReleaseAfterSeconds: env.get('GC_EMPTY_RELEASE_SECONDS', 2 * 60 * 60), + + /** Bulk deletes run in batches of this size, for at most this long per sweep. */ + deleteBatch: 20_000, + deleteBudgetMs: env.get('GC_DELETE_BUDGET_MS', 3 * 60 * 1000), + + /** + * Deferred files (CONCEPT.md D28) are deleted this long after they were + * first seen, processed or not. The articles stay on the server. + */ + deferredRetentionDays: env.get('GC_DEFERRED_RETENTION_DAYS', 30), + + /** Redis hashes from the old header worker (before D12); they only need to expire. */ + legacyPendingFiles: { + prefix: 'file:', + ttlSeconds: 48 * 60 * 60, + }, +} diff --git a/config/indexer.ts b/config/indexer.ts new file mode 100644 index 0000000..c3a2d29 --- /dev/null +++ b/config/indexer.ts @@ -0,0 +1,47 @@ +import env from '#start/env' + +export default { + /** How often `index:scheduler` checks the groups for new articles. */ + intervalMs: env.get('INDEX_INTERVAL_MS', 60 * 1000), + /** Articles per fetch job (one XOVER call). */ + batchSize: BigInt(env.get('INDEX_BATCH_SIZE', 20_000)), + /** Upper bound on fetch jobs queued per group per tick, so catching up is spread out. */ + maxBatchesPerTick: env.get('INDEX_MAX_BATCHES_PER_TICK', 5), + /** + * Where a group that was never indexed starts: this many articles before + * the newest one. The server's low-water mark can be years back. + */ + initialBackfill: BigInt(env.get('INDEX_INITIAL_BACKFILL', 50_000)), + + /** + * Obfuscated articles whose message-id matches one of these are not sent + * to the body queue: their uploader spreads every file over many groups, + * so identifying them would take far more bandwidth than there is + * (see LOG.md, 2026-09-28 (9)). Normal-subject articles are unaffected. + */ + skipBodyMessageIds: [ + // hex7$hex7$hex7@hex10.hex5, ~60k articles/min over 10+ groups + /^?$/, + ], + + /** + * Obfuscated articles whose message-id matches one of these don't go to + * the body queue. Instead they're recorded per file in `deferred_files` + * (their subject keeps the file name, only the part counter is missing). + * `worker:deferred` later fetches one body per file (CONCEPT.md D28). + */ + deferBodyMessageIds: [ + // ngPost: . Mostly RAR5 with encrypted headers (LOG.md 2026-09-28 (14)). + /^?$/, + ], + /** + * Spotnet header groups (CONCEPT.md D29): their headers are spots, not + * binaries. Add one with `group:add free.pt --backfill 5000`. + */ + spotGroups: ['free.pt'], + + /** worker:deferred leaves a file alone until it has had no new articles for this long. */ + deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000), + /** Files of at most this many parts are downloaded completely (small archives, NZBs, par2 index files). */ + deferredExpandMaxParts: env.get('DEFERRED_EXPAND_MAX_PARTS', 8), +} diff --git a/config/nntp.ts b/config/nntp.ts index 625eebf..add1613 100644 --- a/config/nntp.ts +++ b/config/nntp.ts @@ -6,4 +6,6 @@ export default { user: env.get('NNTP_USER'), password: env.get('NNTP_PASSWORD'), secure: env.get('NNTP_SECURE', false), + /** Connections per process. Mind the provider's limit across all processes. */ + poolSize: env.get('NNTP_POOL_SIZE', 10), } diff --git a/config/static.ts b/config/static.ts new file mode 100644 index 0000000..8d2878c --- /dev/null +++ b/config/static.ts @@ -0,0 +1,17 @@ +import { defineConfig } from '@adonisjs/static' + +/** + * Configuration options to tweak the static files middleware. + * The complete set of options are documented on the + * official documentation website. + * + * https://docs.adonisjs.com/guides/static-assets + */ +const staticServerConfig = defineConfig({ + enabled: true, + etag: true, + lastModified: true, + dotFiles: 'ignore', +}) + +export default staticServerConfig \ No newline at end of file diff --git a/database/migrations/1768620764720_create_binaries_table.ts b/database/migrations/1768620764720_create_binaries_table.ts new file mode 100644 index 0000000..6054976 --- /dev/null +++ b/database/migrations/1768620764720_create_binaries_table.ts @@ -0,0 +1,32 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Files that are still being collected from XOVER. One row per uploaded file, + * identified by file_key (see lib/subject/file_key.ts). + */ +export default class extends BaseSchema { + protected tableName = 'binaries' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.bigIncrements('id') + table.string('file_key', 40).notNullable().unique() + table.text('poster').notNullable() + table.text('subject').notNullable() + table.text('name').notNullable() + table.integer('total_parts').notNullable() + table.jsonb('groups').notNullable() + table.timestamp('posted_at', { useTz: true }) + table.timestamp('first_seen_at', { useTz: true }).notNullable() + table.timestamp('last_seen_at', { useTz: true }).notNullable().index() + table.timestamp('completed_at', { useTz: true }) + // Set when two different articles claim the same part number (see IngestService). + table.timestamp('conflicted_at', { useTz: true }) + table.integer('file_id').unsigned().references('files.id').onDelete('SET NULL') + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/database/migrations/1768620764730_create_parts_table.ts b/database/migrations/1768620764730_create_parts_table.ts new file mode 100644 index 0000000..0470943 --- /dev/null +++ b/database/migrations/1768620764730_create_parts_table.ts @@ -0,0 +1,24 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Segments (articles) of a binary. The primary key makes re-ingesting the + * same range a no-op. + */ +export default class extends BaseSchema { + protected tableName = 'parts' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.bigInteger('binary_id').notNullable().references('binaries.id').onDelete('CASCADE') + table.integer('number').notNullable() + table.text('message_id').notNullable() + table.integer('bytes').notNullable() + table.bigInteger('article_number') + table.primary(['binary_id', 'number']) + }) + } + + async down() { + this.schema.dropTable(this.tableName) + } +} diff --git a/database/migrations/1768620764740_add_file_key_to_files_table.ts b/database/migrations/1768620764740_add_file_key_to_files_table.ts new file mode 100644 index 0000000..8b31535 --- /dev/null +++ b/database/migrations/1768620764740_add_file_key_to_files_table.ts @@ -0,0 +1,17 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +export default class extends BaseSchema { + protected tableName = 'files' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + table.string('file_key', 40).unique() + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => { + table.dropColumn('file_key') + }) + } +} diff --git a/database/migrations/1768620764750_create_releases_table.ts b/database/migrations/1768620764750_create_releases_table.ts new file mode 100644 index 0000000..41a6b58 --- /dev/null +++ b/database/migrations/1768620764750_create_releases_table.ts @@ -0,0 +1,38 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * A release is one download: the files of one post (archive volumes, par2 + * set, nfo). See lib/release/release_key.ts and CONCEPT.md (D16). + */ +export default class extends BaseSchema { + protected tableName = 'releases' + + async up() { + this.schema.createTable(this.tableName, (table) => { + table.bigIncrements('id') + table.string('release_key', 40).notNullable().unique() + table.text('name').notNullable() + // Null for obfuscated posts, whose poster is random per article. + table.text('poster') + // Number of files announced by the [i/M] subject counter, if any. + table.integer('file_total') + table.jsonb('groups').notNullable() + table.timestamp('posted_at', { useTz: true }) + table.timestamp('first_seen_at', { useTz: true }).notNullable() + table.timestamp('last_seen_at', { useTz: true }).notNullable() + }) + + this.schema.alterTable('binaries', (table) => { + table.bigInteger('release_id').references('releases.id').onDelete('SET NULL').index() + }) + this.schema.alterTable('files', (table) => { + table.bigInteger('release_id').references('releases.id').onDelete('SET NULL').index() + }) + } + + async down() { + this.schema.alterTable('files', (table) => table.dropColumn('release_id')) + this.schema.alterTable('binaries', (table) => table.dropColumn('release_id')) + this.schema.dropTable(this.tableName) + } +} diff --git a/database/migrations/1768620764760_add_par2_matching.ts b/database/migrations/1768620764760_add_par2_matching.ts new file mode 100644 index 0000000..11526dd --- /dev/null +++ b/database/migrations/1768620764760_add_par2_matching.ts @@ -0,0 +1,33 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Par2-based grouping of obfuscated posts (CONCEPT.md D16): the par2 file + * list of a release, and the exact file size of body-derived binaries to + * match against it. + */ +export default class extends BaseSchema { + async up() { + this.schema.alterTable('binaries', (table) => { + // Exact file size from =ybegin; only known for body-derived binaries. + table.bigInteger('size').index() + }) + this.schema.alterTable('releases', (table) => { + // How the release was recognised: subject, stem, weak-stem or par2. + table.string('source', 16) + }) + this.schema.createTable('release_files', (table) => { + table.bigInteger('release_id').notNullable().references('releases.id').onDelete('CASCADE') + table.string('par2_file_id', 32).notNullable() + table.text('name').notNullable() + table.bigInteger('size').notNullable().index() + table.string('md5_16k', 32).notNullable() + table.primary(['release_id', 'par2_file_id']) + }) + } + + async down() { + this.schema.dropTable('release_files') + this.schema.alterTable('releases', (table) => table.dropColumn('source')) + this.schema.alterTable('binaries', (table) => table.dropColumn('size')) + } +} diff --git a/database/migrations/1768620764770_create_fetch_ranges_table.ts b/database/migrations/1768620764770_create_fetch_ranges_table.ts new file mode 100644 index 0000000..d09c766 --- /dev/null +++ b/database/migrations/1768620764770_create_fetch_ranges_table.ts @@ -0,0 +1,37 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Article ranges handed to worker:fetch, so a range only counts as indexed + * once it was fetched successfully (CONCEPT.md D18). + */ +export default class extends BaseSchema { + async up() { + this.schema.alterTable('groups', (table) => { + // How far the scheduler has queued; last_indexed_id becomes the + // watermark below which every article was fetched. + table.bigInteger('last_queued_id').notNullable().defaultTo(0) + }) + this.defer(async (db) => { + await db.rawQuery('UPDATE groups SET last_queued_id = last_indexed_id') + }) + + this.schema.createTable('fetch_ranges', (table) => { + table.bigIncrements('id') + table.integer('group_id').notNullable().references('groups.id').onDelete('CASCADE') + table.bigInteger('start_id').notNullable() + table.bigInteger('end_id').notNullable() + // queued | done | failed + table.string('status', 8).notNullable().defaultTo('queued') + table.integer('requeues').notNullable().defaultTo(0) + table.text('error') + table.timestamp('created_at', { useTz: true }).notNullable() + table.timestamp('updated_at', { useTz: true }).notNullable() + table.index(['group_id', 'status']) + }) + } + + async down() { + this.schema.dropTable('fetch_ranges') + this.schema.alterTable('groups', (table) => table.dropColumn('last_queued_id')) + } +} diff --git a/database/migrations/1768620764780_add_archive_contents.ts b/database/migrations/1768620764780_add_archive_contents.ts new file mode 100644 index 0000000..1726d57 --- /dev/null +++ b/database/migrations/1768620764780_add_archive_contents.ts @@ -0,0 +1,25 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** Archive listings from the first RAR volume (CONCEPT.md D19). */ +export default class extends BaseSchema { + async up() { + this.schema.alterTable('files', (table) => { + // RarInfo of a first volume: format, flags and entries. + table.jsonb('archive') + }) + this.schema.alterTable('releases', (table) => { + // Password protected: encrypted file data or encrypted headers. + table.boolean('encrypted') + // Entries of the release's first archive volume. + table.jsonb('contents') + }) + } + + async down() { + this.schema.alterTable('releases', (table) => { + table.dropColumn('contents') + table.dropColumn('encrypted') + }) + this.schema.alterTable('files', (table) => table.dropColumn('archive')) + } +} diff --git a/database/migrations/1768620764790_create_part_conflicts_table.ts b/database/migrations/1768620764790_create_part_conflicts_table.ts new file mode 100644 index 0000000..feebc88 --- /dev/null +++ b/database/migrations/1768620764790_create_part_conflicts_table.ts @@ -0,0 +1,32 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Splitting equal-sized obfuscated volumes (CONCEPT.md D20): the extra + * articles of conflicted binaries are kept, and the split-off volumes point + * to the conflicted binary they came from. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('part_conflicts', (table) => { + table.bigInteger('binary_id').notNullable().references('binaries.id').onDelete('CASCADE') + table.integer('number').notNullable() + table.text('message_id').notNullable() + table.integer('bytes').notNullable() + table.bigInteger('article_number') + table.primary(['binary_id', 'number', 'message_id']) + }) + this.schema.alterTable('binaries', (table) => { + table.bigInteger('parent_binary_id').references('binaries.id').onDelete('SET NULL').index() + // Last time the splitter processed this conflicted binary. + table.timestamp('split_at', { useTz: true }) + }) + } + + async down() { + this.schema.alterTable('binaries', (table) => { + table.dropColumn('split_at') + table.dropColumn('parent_binary_id') + }) + this.schema.dropTable('part_conflicts') + } +} diff --git a/database/migrations/1768620764800_add_md5_to_release_files.ts b/database/migrations/1768620764800_add_md5_to_release_files.ts new file mode 100644 index 0000000..1bfeade --- /dev/null +++ b/database/migrations/1768620764800_add_md5_to_release_files.ts @@ -0,0 +1,16 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** Full-file MD5 from par2, to verify assembled (e.g. split) volumes. */ +export default class extends BaseSchema { + protected tableName = 'release_files' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + table.string('md5', 32) + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => table.dropColumn('md5')) + } +} diff --git a/database/migrations/1768620764810_exclude_legacy_conflicts_from_split.ts b/database/migrations/1768620764810_exclude_legacy_conflicts_from_split.ts new file mode 100644 index 0000000..c4fc3fc --- /dev/null +++ b/database/migrations/1768620764810_exclude_legacy_conflicts_from_split.ts @@ -0,0 +1,36 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Binaries that were flagged conflicted before part_conflicts existed lost + * their extra articles, so splitting them only reshuffles the first article + * per part number, a mix of different volumes. Mark them as never to be + * split, and remove the volumes and files the first splitter run made from + * them (see LOG.md, 2026-09-28). + */ +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + await db.rawQuery(` + WITH boundary AS ( + SELECT migration_time FROM adonis_schema WHERE name LIKE '%1768620764790%' + ), legacy AS ( + SELECT id FROM binaries + WHERE conflicted_at IS NOT NULL AND parent_binary_id IS NULL + AND conflicted_at < (SELECT migration_time FROM boundary) + ), instances AS ( + SELECT id, file_id FROM binaries WHERE parent_binary_id IN (SELECT id FROM legacy) + ), deleted_files AS ( + DELETE FROM files WHERE id IN (SELECT file_id FROM instances WHERE file_id IS NOT NULL) + ), deleted_instances AS ( + DELETE FROM binaries WHERE id IN (SELECT id FROM instances) + ) + -- 'infinity': last_seen_at > split_at is never true, so SplitService skips them. + UPDATE binaries SET split_at = 'infinity' WHERE id IN (SELECT id FROM legacy) + `) + }) + } + + async down() { + // The removed volumes were unverifiable; nothing to restore. + } +} diff --git a/database/migrations/1768620764820_add_crcs_to_parts.ts b/database/migrations/1768620764820_add_crcs_to_parts.ts new file mode 100644 index 0000000..c4a5aa1 --- /dev/null +++ b/database/migrations/1768620764820_add_crcs_to_parts.ts @@ -0,0 +1,30 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * yEnc CRCs of body-derived articles, so split volumes can be verified + * without downloading them (CONCEPT.md D20): the combined part CRCs of a + * volume must equal the whole-file crc32 in its last part. + */ +export default class extends BaseSchema { + async up() { + for (const tableName of ['parts', 'part_conflicts']) { + this.schema.alterTable(tableName, (table) => { + table.string('pcrc32', 8) + // Decoded length of the part (=ypart end - begin + 1). + table.integer('part_size') + // Whole-file CRC32, when the poster included it (usually in the last part). + table.string('file_crc32', 8) + }) + } + } + + async down() { + for (const tableName of ['parts', 'part_conflicts']) { + this.schema.alterTable(tableName, (table) => { + table.dropColumn('pcrc32') + table.dropColumn('part_size') + table.dropColumn('file_crc32') + }) + } + } +} diff --git a/database/migrations/1768620764830_add_content_names.ts b/database/migrations/1768620764830_add_content_names.ts new file mode 100644 index 0000000..5d1aef7 --- /dev/null +++ b/database/migrations/1768620764830_add_content_names.ts @@ -0,0 +1,40 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Names and groupings from file contents (CONCEPT.md D21): a fingerprint + * (exact size + MD5 of the first 16 KiB, as par2 records it) and the real + * filename recovered from par2. + */ +export default class extends BaseSchema { + async up() { + this.schema.alterTable('binaries', (table) => { + table.string('md5_16k', 32).index() + // Filename recovered from file contents (par2); `name` stays as posted. + table.text('real_name') + }) + this.schema.alterTable('files', (table) => { + table.text('real_name') + }) + for (const tableName of ['parts', 'part_conflicts']) { + this.schema.alterTable(tableName, (table) => { + // Only for part 1: MD5 of the first 16 KiB of the file. + table.string('md5_16k', 32) + }) + } + this.schema.alterTable('release_files', (table) => { + table.index(['md5_16k']) + }) + } + + async down() { + this.schema.alterTable('release_files', (table) => table.dropIndex(['md5_16k'])) + for (const tableName of ['parts', 'part_conflicts']) { + this.schema.alterTable(tableName, (table) => table.dropColumn('md5_16k')) + } + this.schema.alterTable('files', (table) => table.dropColumn('real_name')) + this.schema.alterTable('binaries', (table) => { + table.dropColumn('real_name') + table.dropColumn('md5_16k') + }) + } +} diff --git a/database/migrations/1768620764840_add_search_indexes.ts b/database/migrations/1768620764840_add_search_indexes.ts new file mode 100644 index 0000000..b02ffda --- /dev/null +++ b/database/migrations/1768620764840_add_search_indexes.ts @@ -0,0 +1,43 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Web interface (CONCEPT.md D22): trigram indexes for substring search, and + * SQL versions of the "readable name" rule used for filtering. + */ +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + await db.rawQuery('CREATE EXTENSION IF NOT EXISTS pg_trgm') + // Per-file suffixes stripped, as in lib/release fileStem. + await db.rawQuery(` + CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$ + SELECT regexp_replace(name, + '(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|gz|xz|mkv|mp4|avi|m4v|ts|iso|img|flac|mp3|epub|pdf|jpg|png))+$', + '', 'i') + $$`) + // Not an obfuscated token (as in ArchiveService.looksRandom). + await db.rawQuery(` + CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$ + SELECT name IS NOT NULL AND NOT ( + (name_stem(name) ~ '^[A-Za-z0-9]{16,}$' AND name_stem(name) ~ '\\d' + AND name_stem(name) ~ '[a-z]' AND name_stem(name) ~ '[A-Z]') + OR name_stem(name) ~* '^[0-9a-f]{16,}$') + $$`) + await db.rawQuery('CREATE INDEX IF NOT EXISTS releases_name_trgm ON releases USING gin (name gin_trgm_ops)') + await db.rawQuery('CREATE INDEX IF NOT EXISTS files_filename_trgm ON files USING gin (filename gin_trgm_ops)') + await db.rawQuery('CREATE INDEX IF NOT EXISTS files_real_name_trgm ON files USING gin (real_name gin_trgm_ops)') + await db.rawQuery('CREATE INDEX IF NOT EXISTS releases_posted_at_idx ON releases (posted_at DESC NULLS LAST)') + }) + } + + async down() { + this.defer(async (db) => { + await db.rawQuery('DROP INDEX IF EXISTS releases_posted_at_idx') + await db.rawQuery('DROP INDEX IF EXISTS files_real_name_trgm') + await db.rawQuery('DROP INDEX IF EXISTS files_filename_trgm') + await db.rawQuery('DROP INDEX IF EXISTS releases_name_trgm') + await db.rawQuery('DROP FUNCTION IF EXISTS is_readable_name(text)') + await db.rawQuery('DROP FUNCTION IF EXISTS name_stem(text)') + }) + } +} diff --git a/database/migrations/1768620764850_add_release_totals.ts b/database/migrations/1768620764850_add_release_totals.ts new file mode 100644 index 0000000..7f9b0cf --- /dev/null +++ b/database/migrations/1768620764850_add_release_totals.ts @@ -0,0 +1,56 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Per-release totals for browsing and sorting (CONCEPT.md D22). A trigger on + * `files` keeps `releases.file_count` and `releases.size` current, whichever + * code path assigns files to releases. + */ +export default class extends BaseSchema { + async up() { + this.schema.alterTable('files', (table) => { + table.bigInteger('size') + }) + this.schema.alterTable('releases', (table) => { + table.integer('file_count').notNullable().defaultTo(0).index() + table.bigInteger('size').notNullable().defaultTo(0).index() + }) + this.defer(async (db) => { + await db.rawQuery(` + UPDATE files f SET size = s.size + FROM (SELECT id, (SELECT coalesce(sum((v->>'size')::bigint), 0) FROM jsonb_each(message_ids) e(k, v)) AS size FROM files) s + WHERE s.id = f.id`) + await db.rawQuery(` + UPDATE releases r SET file_count = t.n, size = t.size + FROM (SELECT release_id, count(*) AS n, coalesce(sum(size), 0) AS size FROM files + WHERE release_id IS NOT NULL GROUP BY release_id) t + WHERE t.release_id = r.id`) + await db.rawQuery(` + CREATE OR REPLACE FUNCTION files_release_totals() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP IN ('UPDATE', 'DELETE') AND OLD.release_id IS NOT NULL THEN + UPDATE releases SET file_count = file_count - 1, size = size - coalesce(OLD.size, 0) WHERE id = OLD.release_id; + END IF; + IF TG_OP IN ('UPDATE', 'INSERT') AND NEW.release_id IS NOT NULL THEN + UPDATE releases SET file_count = file_count + 1, size = size + coalesce(NEW.size, 0) WHERE id = NEW.release_id; + END IF; + RETURN NULL; + END $$`) + await db.rawQuery(` + CREATE TRIGGER files_release_totals + AFTER INSERT OR DELETE OR UPDATE OF release_id, size ON files + FOR EACH ROW EXECUTE FUNCTION files_release_totals()`) + }) + } + + async down() { + this.defer(async (db) => { + await db.rawQuery('DROP TRIGGER IF EXISTS files_release_totals ON files') + await db.rawQuery('DROP FUNCTION IF EXISTS files_release_totals()') + }) + this.schema.alterTable('releases', (table) => { + table.dropColumn('size') + table.dropColumn('file_count') + }) + this.schema.alterTable('files', (table) => table.dropColumn('size')) + } +} diff --git a/database/migrations/1768620764860_add_release_completeness.ts b/database/migrations/1768620764860_add_release_completeness.ts new file mode 100644 index 0000000..a50c2cb --- /dev/null +++ b/database/migrations/1768620764860_add_release_completeness.ts @@ -0,0 +1,107 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Stored completeness for the "Complete" filter (CONCEPT.md D22); computing + * it per request took ~30 s on 34k releases. Triggers keep it current when + * files, par2 lists, sizes or announced totals change. + */ +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + await db.rawQuery('CREATE INDEX IF NOT EXISTS binaries_file_id_idx ON binaries (file_id)') + await db.rawQuery('CREATE INDEX IF NOT EXISTS releases_contents_trgm ON releases USING gin ((contents::text) gin_trgm_ops)') + await db.rawQuery('ALTER TABLE releases ADD COLUMN complete boolean NOT NULL DEFAULT false') + await db.rawQuery('CREATE INDEX releases_complete_idx ON releases (complete)') + + // All files announced by [i/M] are there, or every par2-listed file + // (>= 64 KiB) is present by exact size or real name. + await db.rawQuery(` + CREATE OR REPLACE FUNCTION release_complete(rid bigint) RETURNS boolean STABLE LANGUAGE sql AS $$ + SELECT coalesce(( + SELECT CASE + WHEN r.file_total IS NOT NULL THEN r.file_count >= r.file_total + WHEN EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id) THEN NOT EXISTS ( + SELECT 1 FROM release_files rf + WHERE rf.release_id = r.id AND rf.size >= 65536 + AND NOT EXISTS ( + SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id + WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) + ELSE false END + FROM releases r WHERE r.id = rid), false) + $$`) + + await db.rawQuery(` + CREATE OR REPLACE FUNCTION refresh_release_complete(rid bigint) RETURNS void LANGUAGE sql AS $$ + UPDATE releases SET complete = release_complete(rid) WHERE id = rid AND complete IS DISTINCT FROM release_complete(rid) + $$`) + + // files: after the totals trigger has updated file_count (trigger names sort alphabetically). + await db.rawQuery(` + CREATE OR REPLACE FUNCTION files_release_complete() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF TG_OP IN ('UPDATE', 'DELETE') AND OLD.release_id IS NOT NULL THEN PERFORM refresh_release_complete(OLD.release_id); END IF; + IF TG_OP IN ('UPDATE', 'INSERT') AND NEW.release_id IS NOT NULL THEN PERFORM refresh_release_complete(NEW.release_id); END IF; + RETURN NULL; + END $$`) + await db.rawQuery(` + CREATE TRIGGER files_zz_release_complete + AFTER INSERT OR DELETE OR UPDATE OF release_id, size, real_name ON files + FOR EACH ROW EXECUTE FUNCTION files_release_complete()`) + + await db.rawQuery(` + CREATE OR REPLACE FUNCTION release_files_release_complete() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM refresh_release_complete(NEW.release_id); + RETURN NULL; + END $$`) + await db.rawQuery(` + CREATE TRIGGER release_files_release_complete + AFTER INSERT ON release_files FOR EACH ROW EXECUTE FUNCTION release_files_release_complete()`) + + // An exact size from content inspection can make a par2-listed file "present". + await db.rawQuery(` + CREATE OR REPLACE FUNCTION binaries_release_complete() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + IF NEW.file_id IS NOT NULL THEN + PERFORM refresh_release_complete(f.release_id) FROM files f WHERE f.id = NEW.file_id AND f.release_id IS NOT NULL; + END IF; + RETURN NULL; + END $$`) + await db.rawQuery(` + CREATE TRIGGER binaries_release_complete + AFTER UPDATE OF size ON binaries FOR EACH ROW + WHEN (OLD.size IS DISTINCT FROM NEW.size) EXECUTE FUNCTION binaries_release_complete()`) + + await db.rawQuery(` + CREATE OR REPLACE FUNCTION releases_file_total_complete() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM refresh_release_complete(NEW.id); + RETURN NULL; + END $$`) + await db.rawQuery(` + CREATE TRIGGER releases_file_total_complete + AFTER UPDATE OF file_total ON releases FOR EACH ROW + WHEN (OLD.file_total IS DISTINCT FROM NEW.file_total) EXECUTE FUNCTION releases_file_total_complete()`) + + await db.rawQuery('UPDATE releases SET complete = release_complete(id) WHERE file_count > 0') + }) + } + + async down() { + this.defer(async (db) => { + await db.rawQuery('DROP TRIGGER IF EXISTS releases_file_total_complete ON releases') + await db.rawQuery('DROP TRIGGER IF EXISTS binaries_release_complete ON binaries') + await db.rawQuery('DROP TRIGGER IF EXISTS release_files_release_complete ON release_files') + await db.rawQuery('DROP TRIGGER IF EXISTS files_zz_release_complete ON files') + await db.rawQuery('DROP FUNCTION IF EXISTS releases_file_total_complete()') + await db.rawQuery('DROP FUNCTION IF EXISTS binaries_release_complete()') + await db.rawQuery('DROP FUNCTION IF EXISTS release_files_release_complete()') + await db.rawQuery('DROP FUNCTION IF EXISTS files_release_complete()') + await db.rawQuery('DROP FUNCTION IF EXISTS refresh_release_complete(bigint)') + await db.rawQuery('DROP FUNCTION IF EXISTS release_complete(bigint)') + await db.rawQuery('ALTER TABLE releases DROP COLUMN IF EXISTS complete') + await db.rawQuery('DROP INDEX IF EXISTS releases_contents_trgm') + await db.rawQuery('DROP INDEX IF EXISTS binaries_file_id_idx') + }) + } +} diff --git a/database/migrations/1768620764870_strict_par2_matching.ts b/database/migrations/1768620764870_strict_par2_matching.ts new file mode 100644 index 0000000..47533da --- /dev/null +++ b/database/migrations/1768620764870_strict_par2_matching.ts @@ -0,0 +1,28 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * par2 set id per listed file (traceability; see LOG.md 2026-09-28 for the + * cross-post merges it helps diagnose), and a stricter "random name" rule: + * long letter-only tokens with many capitals count as random too + * (e.g. RmISakytFwXSCEzjVEpaKk). + */ +export default class extends BaseSchema { + async up() { + this.schema.alterTable('release_files', (table) => { + table.string('set_id', 32) + }) + this.defer(async (db) => { + await db.rawQuery(` + CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$ + SELECT name IS NOT NULL AND NOT ( + (name_stem(name) ~ '^[A-Za-z0-9]{16,}$' AND name_stem(name) ~ '[a-z]' AND name_stem(name) ~ '[A-Z]' + AND (name_stem(name) ~ '\\d' OR length(regexp_replace(name_stem(name), '[^A-Z]', '', 'g')) >= 6)) + OR name_stem(name) ~* '^[0-9a-f]{16,}$') + $$`) + }) + } + + async down() { + this.schema.alterTable('release_files', (table) => table.dropColumn('set_id')) + } +} diff --git a/database/migrations/1768620764880_add_release_category.ts b/database/migrations/1768620764880_add_release_category.ts new file mode 100644 index 0000000..a69ded6 --- /dev/null +++ b/database/migrations/1768620764880_add_release_category.ts @@ -0,0 +1,45 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Newznab category per release (CONCEPT.md D23), derived from the name and + * the archive contents. A stored generated column, so it follows renames + * and newly listed contents without any application code. + */ +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + await db.rawQuery(` + CREATE OR REPLACE FUNCTION release_category(name text, contents jsonb) RETURNS integer IMMUTABLE LANGUAGE sql AS $$ + SELECT CASE + -- TV: S01E02, 1x02, or a season pack (S01). + WHEN name ~* '(^|[._ -])(S\\d{1,2}E\\d{1,3}|\\d{1,2}x\\d{2})([._ -]|$)' OR name ~* '[._ -]S\\d{1,2}([._ -]|$)' THEN + CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 5045 + WHEN name ~* '720p|1080p' THEN 5040 ELSE 5030 END + -- Movies: a year plus a source or quality marker. + WHEN name ~* '(^|[._ (-])(19|20)\\d{2}([._ )-]|$)' + AND name ~* '(480p|576p|720p|1080p|2160p|blu-?ray|web-?dl|webrip|bdrip|brrip|dvdrip|hdtv|x26[45]|h\\.?26[45]|hevc|remux)' THEN + CASE WHEN name ~* '2160p|(^|[._ -])(uhd|4k)([._ -]|$)' THEN 2045 + WHEN name ~* '720p|1080p|blu-?ray' THEN 2040 ELSE 2030 END + -- Audio: lossless or mp3 by contents, otherwise scene music names (-WEB-2026-GROUP). + WHEN coalesce(contents::text, '') ~* '\\.flac"' OR name ~* '(^|[._ -])flac([._ -]|$)' THEN 3040 + WHEN coalesce(contents::text, '') ~* '\\.mp3"' OR name ~* '(^|[._ -])mp3([._ -]|$)' + OR name ~* '-(WEB|CD|CDM|CDS|EP|VINYL|VLS|SAT|FM|DAB)-(19|20)\\d{2}-[A-Za-z0-9]+$' THEN 3010 + WHEN coalesce(contents::text, '') ~* '\\.(epub|mobi|azw3)"' OR name ~* '(^|[._ -])(epub|ebook|mobi)([._ -]|$)' THEN 7020 + WHEN coalesce(contents::text, '') ~* '\\.(exe|msi|dmg)"' THEN 4000 + ELSE 8010 + END + $$`) + await db.rawQuery( + 'ALTER TABLE releases ADD COLUMN category integer GENERATED ALWAYS AS (release_category(name, contents)) STORED' + ) + await db.rawQuery('CREATE INDEX releases_category_idx ON releases (category)') + }) + } + + async down() { + this.defer(async (db) => { + await db.rawQuery('ALTER TABLE releases DROP COLUMN IF EXISTS category') + await db.rawQuery('DROP FUNCTION IF EXISTS release_category(text, jsonb)') + }) + } +} diff --git a/database/migrations/1768620764890_ignore_identical_reposts.ts b/database/migrations/1768620764890_ignore_identical_reposts.ts new file mode 100644 index 0000000..6d25194 --- /dev/null +++ b/database/migrations/1768620764890_ignore_identical_reposts.ts @@ -0,0 +1,31 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * A second article with the same part CRC is a repost of the same data, not + * a different file (CONCEPT.md D20). Drop such recorded conflicts and unflag + * binaries whose only conflicts were reposts. Binaries flagged before CRCs + * were stored (split_at = 'infinity') cannot be checked and stay flagged. + */ +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + await db.rawQuery(` + DELETE FROM part_conflicts c USING parts p + WHERE p.binary_id = c.binary_id AND p.number = c.number + AND p.pcrc32 IS NOT NULL AND c.pcrc32 IS NOT NULL AND p.pcrc32 = c.pcrc32`) + // Split volumes made from reposts are copies of their parent's file. + await db.rawQuery(` + DELETE FROM binaries v USING binaries b + WHERE v.parent_binary_id = b.id AND v.completed_at IS NULL + AND b.conflicted_at IS NOT NULL AND b.split_at IS DISTINCT FROM 'infinity' + AND NOT EXISTS (SELECT 1 FROM part_conflicts c WHERE c.binary_id = b.id)`) + await db.rawQuery(` + UPDATE binaries b SET conflicted_at = NULL + WHERE b.conflicted_at IS NOT NULL AND b.parent_binary_id IS NULL + AND b.split_at IS DISTINCT FROM 'infinity' + AND NOT EXISTS (SELECT 1 FROM part_conflicts c WHERE c.binary_id = b.id)`) + }) + } + + async down() {} +} diff --git a/database/migrations/1768620764900_add_fetch_bodies_to_groups.ts b/database/migrations/1768620764900_add_fetch_bodies_to_groups.ts new file mode 100644 index 0000000..94a41e5 --- /dev/null +++ b/database/migrations/1768620764900_add_fetch_bodies_to_groups.ts @@ -0,0 +1,24 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Body downloads per group (CONCEPT.md D24). Identifying obfuscated articles + * costs one body download each; busy groups carry far more of them than the + * line can fetch, so they are indexed header-only (normal subjects) unless + * enabled. + */ +export default class extends BaseSchema { + protected tableName = 'groups' + + async up() { + this.schema.alterTable(this.tableName, (table) => { + table.boolean('fetch_bodies').notNullable().defaultTo(false) + }) + this.defer(async (db) => { + await db.rawQuery(`UPDATE groups SET fetch_bodies = true WHERE name = 'alt.binaries.test'`) + }) + } + + async down() { + this.schema.alterTable(this.tableName, (table) => table.dropColumn('fetch_bodies')) + } +} diff --git a/database/migrations/1768620764910_readable_name_any_extension.ts b/database/migrations/1768620764910_readable_name_any_extension.ts new file mode 100644 index 0000000..fe4a57c --- /dev/null +++ b/database/migrations/1768620764910_readable_name_any_extension.ts @@ -0,0 +1,23 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * is_readable_name(): also judge the stem without an unknown short + * extension, so `326678d1d07ef12a.bin` counts as random (CONCEPT.md D22). + */ +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + await db.rawQuery(` + CREATE OR REPLACE FUNCTION is_readable_name(name text) RETURNS boolean IMMUTABLE LANGUAGE sql AS $$ + WITH s AS (SELECT regexp_replace(name_stem(name), '\\.[A-Za-z0-9]{1,4}$', '') AS stem) + SELECT name IS NOT NULL AND NOT ( + (stem ~ '^[A-Za-z0-9]{16,}$' AND stem ~ '[a-z]' AND stem ~ '[A-Z]' + AND (stem ~ '\\d' OR length(regexp_replace(stem, '[^A-Z]', '', 'g')) >= 6)) + OR stem ~* '^[0-9a-f]{16,}$') + FROM s + $$`) + }) + } + + async down() {} +} diff --git a/database/migrations/1768620764920_add_content_types.ts b/database/migrations/1768620764920_add_content_types.ts new file mode 100644 index 0000000..e46ba71 --- /dev/null +++ b/database/migrations/1768620764920_add_content_types.ts @@ -0,0 +1,22 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * File type by magic bytes (lib/content), and releases that are only an + * encrypted container (CONCEPT.md D25). + */ +export default class extends BaseSchema { + async up() { + this.schema.alterTable('files', (table) => { + table.string('content_type', 16).index() + }) + this.schema.alterTable('releases', (table) => { + // e.g. 'nzbe': an encrypted NZB, useless without its key. + table.string('container', 16) + }) + } + + async down() { + this.schema.alterTable('releases', (table) => table.dropColumn('container')) + this.schema.alterTable('files', (table) => table.dropColumn('content_type')) + } +} diff --git a/database/migrations/1768620764930_add_nzb_import.ts b/database/migrations/1768620764930_add_nzb_import.ts new file mode 100644 index 0000000..7970bad --- /dev/null +++ b/database/migrations/1768620764930_add_nzb_import.ts @@ -0,0 +1,24 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * NZBs posted to Usenet are read into releases (CONCEPT.md D26): the archive + * password they may carry, and a lookup of files by their first segment. + */ +export default class extends BaseSchema { + async up() { + this.schema.alterTable('releases', (table) => { + // From of a posted NZB; passed on in our NZBs. + table.text('password') + }) + this.defer(async (db) => { + await db.rawQuery(`CREATE INDEX IF NOT EXISTS files_first_segment_idx ON files ((message_ids->'1'->>'id'))`) + }) + } + + async down() { + this.defer(async (db) => { + await db.rawQuery('DROP INDEX IF EXISTS files_first_segment_idx') + }) + this.schema.alterTable('releases', (table) => table.dropColumn('password')) + } +} diff --git a/database/migrations/1768620764940_name_stem_subtitles.ts b/database/migrations/1768620764940_name_stem_subtitles.ts new file mode 100644 index 0000000..6990fa6 --- /dev/null +++ b/database/migrations/1768620764940_name_stem_subtitles.ts @@ -0,0 +1,17 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** name_stem(): also strip subtitle extensions, as lib/release fileStem does. */ +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + await db.rawQuery(` + CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$ + SELECT regexp_replace(name, + '(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|gz|xz|mkv|mp4|avi|m4v|ts|iso|img|flac|mp3|epub|pdf|jpg|png|srt|sub|idx|ass|ssa))+$', + '', 'i') + $$`) + }) + } + + async down() {} +} diff --git a/database/migrations/1768620764950_name_stem_split_volumes.ts b/database/migrations/1768620764950_name_stem_split_volumes.ts new file mode 100644 index 0000000..bb92ab5 --- /dev/null +++ b/database/migrations/1768620764950_name_stem_split_volumes.ts @@ -0,0 +1,21 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * name_stem(): strip a 3-digit split suffix only after a file extension + * (x.mkv.001), not codec names like H.264 / H.265, matching lib/release. + */ +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + await db.rawQuery(` + CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$ + SELECT regexp_replace( + regexp_replace(name, '(\\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\\.\\d{3}$', '\\1', 'i'), + '(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z|zip|nfo|sfv|nzb|tar|zst|gz|xz|mkv|mp4|avi|m4v|ts|iso|img|flac|mp3|epub|pdf|jpg|png|srt|sub|idx|ass|ssa))+$', + '', 'i') + $$`) + }) + } + + async down() {} +} diff --git a/database/migrations/1768620764960_name_stem_more_extensions.ts b/database/migrations/1768620764960_name_stem_more_extensions.ts new file mode 100644 index 0000000..2b8b226 --- /dev/null +++ b/database/migrations/1768620764960_name_stem_more_extensions.ts @@ -0,0 +1,18 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** name_stem(): same extension list as lib/release fileStem (adds bin, m3u, cue, log, bz2). */ +export default class extends BaseSchema { + async up() { + this.defer(async (db) => { + await db.rawQuery(` + CREATE OR REPLACE FUNCTION name_stem(name text) RETURNS text IMMUTABLE LANGUAGE sql AS $$ + SELECT regexp_replace( + regexp_replace(name, '(\\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\\.\\d{3}$', '\\1', 'i'), + '(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z|zip|nfo|sfv|nzb|srr|srs|txt|md5|sha1|tar|zst|gz|xz|bz2|mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf|jpg|png|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log))+$', + '', 'i') + $$`) + }) + } + + async down() {} +} diff --git a/database/migrations/1768620764970_create_deferred_files_table.ts b/database/migrations/1768620764970_create_deferred_files_table.ts new file mode 100644 index 0000000..49ece99 --- /dev/null +++ b/database/migrations/1768620764970_create_deferred_files_table.ts @@ -0,0 +1,47 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Files of uploaders whose bodies are not fetched right away (ngPost, see + * CONCEPT.md D28): one row per file, from the headers alone. The row keeps + * the article range, so the file can be found again, and is the queue for + * worker:deferred, which fetches one body per file. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('deferred_files', (table) => { + table.bigIncrements('id') + table.string('file_key', 40).notNullable().unique() + // File name from the subject (random for ngPost). + table.text('name').notNullable() + table.text('subject').notNullable() + table.text('poster').notNullable() + // Article numbers are counted in the group where the file was first seen. + table.text('group_name').notNullable() + table.bigInteger('first_article').notNullable() + table.text('first_message_id').notNullable() + table.bigInteger('last_article').notNullable() + table.integer('article_count').notNullable() + table.bigInteger('bytes').notNullable() + table.timestamp('posted_at', { useTz: true }) + table.timestamp('first_seen_at', { useTz: true }).notNullable() + table.timestamp('last_seen_at', { useTz: true }).notNullable() + // pending → sampled (one body read) or expanded (sent to the body queue) or failed. + table.string('status', 16).notNullable().defaultTo('pending') + table.timestamp('processed_at', { useTz: true }) + // From the sampled body. + table.text('yenc_name') + table.bigInteger('size') + table.integer('total_parts') + table.integer('sampled_part') + table.string('content_type', 16) + table.boolean('encrypted') + table.text('error') + table.index(['status', 'last_seen_at']) + table.index(['first_seen_at']) + }) + } + + async down() { + this.schema.dropTable('deferred_files') + } +} diff --git a/database/migrations/1768620764980_create_deferred_ranges_table.ts b/database/migrations/1768620764980_create_deferred_ranges_table.ts new file mode 100644 index 0000000..125dbb0 --- /dev/null +++ b/database/migrations/1768620764980_create_deferred_ranges_table.ts @@ -0,0 +1,28 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Deferred articles whose headers don't say which file they belong to + * (ngPost with random subjects and a new poster every few articles, see + * CONCEPT.md D28). Only the article range per fetch batch is kept; + * `deferred:expand-ranges` finds the articles again and queues their bodies. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('deferred_ranges', (table) => { + table.bigIncrements('id') + table.text('group_name').notNullable() + table.bigInteger('first_article').notNullable() + table.bigInteger('last_article').notNullable() + table.integer('article_count').notNullable() + table.bigInteger('bytes').notNullable() + table.timestamp('created_at', { useTz: true }).notNullable() + table.timestamp('expanded_at', { useTz: true }) + table.index(['expanded_at', 'id']) + table.index(['created_at']) + }) + } + + async down() { + this.schema.dropTable('deferred_ranges') + } +} diff --git a/database/migrations/1768620764990_create_spots_table.ts b/database/migrations/1768620764990_create_spots_table.ts new file mode 100644 index 0000000..1781f3b --- /dev/null +++ b/database/migrations/1768620764990_create_spots_table.ts @@ -0,0 +1,40 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Spotnet spots (CONCEPT.md D29): signed posts in free.pt that point to a + * compressed NZB. Filled from headers by worker:fetch, then worker:spots + * verifies each spot, reads its NZB and imports it as a release. + */ +export default class extends BaseSchema { + async up() { + this.schema.createTable('spots', (table) => { + table.text('message_id').primary() + table.text('group_name').notNullable() + table.bigInteger('article_number') + table.text('title').notNullable() + table.text('tag') + table.text('poster').notNullable() + table.integer('category') + table.jsonb('subcats') + table.bigInteger('size') + table.timestamp('posted_at', { useTz: true }) + table.integer('key_id') + // Spotweb's spotter id (CRC of the poster's key), to check personal disposes. + table.text('spotter_id') + // pending → imported, or unverified / no-nzb / moderation / removed / failed. + table.string('status', 16).notNullable().defaultTo('pending') + table.bigInteger('release_id').references('releases.id').onDelete('SET NULL') + table.text('website') + table.text('error') + table.timestamp('created_at', { useTz: true }).notNullable() + table.timestamp('processed_at', { useTz: true }) + table.index(['status', 'created_at']) + table.index(['release_id']) + table.index(['status', 'release_id']) + }) + } + + async down() { + this.schema.dropTable('spots') + } +} diff --git a/database/migrations/1768620765000_partial_indexes.ts b/database/migrations/1768620765000_partial_indexes.ts new file mode 100644 index 0000000..9417de1 --- /dev/null +++ b/database/migrations/1768620765000_partial_indexes.ts @@ -0,0 +1,44 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Smaller indexes (LOG.md 2026-09-28 (17)). 99.5% of releases have no files + * yet (most never will), and search only looks at releases with files; most + * binaries have no size, fingerprint, file or parent. Indexing only the rows + * that queries can match saves GBs and work on every insert. Built + * concurrently, so ingest keeps running. + */ +const INDEXES: [table: string, name: string, definition: string][] = [ + ['releases', 'releases_name_trgm', `USING gin (name gin_trgm_ops) WHERE file_count > 0`], + ['releases', 'releases_contents_trgm', `USING gin ((contents::text) gin_trgm_ops) WHERE file_count > 0`], + ['releases', 'releases_posted_at_idx', `(posted_at DESC NULLS LAST) WHERE file_count > 0`], + ['releases', 'releases_size_index', `(size) WHERE file_count > 0`], + ['releases', 'releases_file_count_index', `(file_count) WHERE file_count > 0`], + ['releases', 'releases_complete_idx', `(complete) WHERE file_count > 0`], + ['releases', 'releases_category_idx', `(category) WHERE file_count > 0`], + ['binaries', 'binaries_md5_16k_index', `(md5_16k) WHERE md5_16k IS NOT NULL`], + ['binaries', 'binaries_size_index', `(size) WHERE size IS NOT NULL`], + ['binaries', 'binaries_file_id_idx', `(file_id) WHERE file_id IS NOT NULL`], + ['binaries', 'binaries_parent_binary_id_index', `(parent_binary_id) WHERE parent_binary_id IS NOT NULL`], +] + +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + for (const [table, name, definition] of INDEXES) { + // Build the partial index under a temporary name, then swap it in. + await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}_partial`) + await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name}_partial ON ${table} ${definition}`) + await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`) + await this.db.rawQuery(`ALTER INDEX ${name}_partial RENAME TO ${name}`) + } + } + + async down() { + for (const [table, name, definition] of INDEXES) { + const full = definition.replace(/ WHERE .*$/, '') + await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`) + await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name} ON ${table} ${full}`) + } + } +} diff --git a/database/migrations/1768620765010_spot_listings.ts b/database/migrations/1768620765010_spot_listings.ts new file mode 100644 index 0000000..d22f979 --- /dev/null +++ b/database/migrations/1768620765010_spot_listings.ts @@ -0,0 +1,41 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Spot listings (CONCEPT.md D29): old spots become a release without files + * (source `spot`), whose NZB is fetched when first downloaded. Search shows + * releases with files or spot listings, so the partial release indexes + * cover both. Spots are verified from their overview when fetched. + */ +const VISIBLE = `(file_count > 0 OR source = 'spot')` +const INDEXES: [name: string, definition: string][] = [ + ['releases_name_trgm', `USING gin (name gin_trgm_ops)`], + ['releases_contents_trgm', `USING gin ((contents::text) gin_trgm_ops)`], + ['releases_posted_at_idx', `(posted_at DESC NULLS LAST)`], + ['releases_size_index', `(size)`], + ['releases_file_count_index', `(file_count)`], + ['releases_complete_idx', `(complete)`], + ['releases_category_idx', `(category)`], +] + +export default class extends BaseSchema { + static disableTransactions = true + + async up() { + await this.db.rawQuery(`ALTER TABLE spots ADD COLUMN IF NOT EXISTS verified boolean`) + await this.swap(VISIBLE) + } + + async down() { + await this.swap('file_count > 0') + await this.db.rawQuery(`ALTER TABLE spots DROP COLUMN IF EXISTS verified`) + } + + private async swap(predicate: string) { + for (const [name, definition] of INDEXES) { + await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}_next`) + await this.db.rawQuery(`CREATE INDEX CONCURRENTLY ${name}_next ON releases ${definition} WHERE ${predicate}`) + await this.db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS ${name}`) + await this.db.rawQuery(`ALTER INDEX ${name}_next RENAME TO ${name}`) + } + } +} diff --git a/database/schema.ts b/database/schema.ts index 7078378..1c76e8d 100644 --- a/database/schema.ts +++ b/database/schema.ts @@ -32,14 +32,158 @@ export class AuthAccessTokenSchema extends BaseModel { declare updatedAt: DateTime | null } +export class BinarySchema extends BaseModel { + static $columns = ['completedAt', 'conflictedAt', 'fileId', 'fileKey', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'md516K', 'name', 'parentBinaryId', 'postedAt', 'poster', 'realName', 'releaseId', 'size', 'splitAt', 'subject', 'totalParts'] as const + $columns = BinarySchema.$columns + @column.dateTime() + declare completedAt: DateTime | null + @column.dateTime() + declare conflictedAt: DateTime | null + @column() + declare fileId: number | null + @column() + declare fileKey: string + @column.dateTime() + declare firstSeenAt: DateTime + @column() + declare groups: any + @column({ isPrimary: true }) + declare id: bigint | number + @column.dateTime() + declare lastSeenAt: DateTime + @column() + declare md516K: string | null + @column() + declare name: string + @column() + declare parentBinaryId: bigint | number | null + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string + @column() + declare realName: string | null + @column() + declare releaseId: bigint | number | null + @column() + declare size: bigint | number | null + @column.dateTime() + declare splitAt: DateTime | null + @column() + declare subject: string + @column() + declare totalParts: number +} + +export class DeferredFileSchema extends BaseModel { + static $columns = ['articleCount', 'bytes', 'contentType', 'encrypted', 'error', 'fileKey', 'firstArticle', 'firstMessageId', 'firstSeenAt', 'groupName', 'id', 'lastArticle', 'lastSeenAt', 'name', 'postedAt', 'poster', 'processedAt', 'sampledPart', 'size', 'status', 'subject', 'totalParts', 'yencName'] as const + $columns = DeferredFileSchema.$columns + @column() + declare articleCount: number + @column() + declare bytes: bigint | number + @column() + declare contentType: string | null + @column() + declare encrypted: boolean | null + @column() + declare error: string | null + @column() + declare fileKey: string + @column() + declare firstArticle: bigint | number + @column() + declare firstMessageId: string + @column.dateTime() + declare firstSeenAt: DateTime + @column() + declare groupName: string + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare lastArticle: bigint | number + @column.dateTime() + declare lastSeenAt: DateTime + @column() + declare name: string + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string + @column.dateTime() + declare processedAt: DateTime | null + @column() + declare sampledPart: number | null + @column() + declare size: bigint | number | null + @column() + declare status: string + @column() + declare subject: string + @column() + declare totalParts: number | null + @column() + declare yencName: string | null +} + +export class DeferredRangeSchema extends BaseModel { + static $columns = ['articleCount', 'bytes', 'createdAt', 'expandedAt', 'firstArticle', 'groupName', 'id', 'lastArticle'] as const + $columns = DeferredRangeSchema.$columns + @column() + declare articleCount: number + @column() + declare bytes: bigint | number + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column.dateTime() + declare expandedAt: DateTime | null + @column() + declare firstArticle: bigint | number + @column() + declare groupName: string + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare lastArticle: bigint | number +} + +export class FetchRangeSchema extends BaseModel { + static $columns = ['createdAt', 'endId', 'error', 'groupId', 'id', 'requeues', 'startId', 'status', 'updatedAt'] as const + $columns = FetchRangeSchema.$columns + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare endId: bigint | number + @column() + declare error: string | null + @column() + declare groupId: number + @column({ isPrimary: true }) + declare id: bigint | number + @column() + declare requeues: number + @column() + declare startId: bigint | number + @column() + declare status: string + @column.dateTime({ autoCreate: true, autoUpdate: true }) + declare updatedAt: DateTime +} + export class FileSchema extends BaseModel { - static $columns = ['createdAt', 'date', 'filename', 'groups', 'id', 'messageIds', 'parts', 'poster', 'updatedAt'] as const + static $columns = ['archive', 'contentType', 'createdAt', 'date', 'fileKey', 'filename', 'groups', 'id', 'messageIds', 'parts', 'poster', 'realName', 'releaseId', 'size', 'updatedAt'] as const $columns = FileSchema.$columns + @column() + declare archive: any | null + @column() + declare contentType: string | null @column.dateTime({ autoCreate: true }) declare createdAt: DateTime | null @column() declare date: bigint | number @column() + declare fileKey: string | null + @column() declare filename: string @column() declare groups: any | null @@ -51,27 +195,184 @@ export class FileSchema extends BaseModel { declare parts: number @column() declare poster: string + @column() + declare realName: string | null + @column() + declare releaseId: bigint | number | null + @column() + declare size: bigint | number | null @column.dateTime({ autoCreate: true, autoUpdate: true }) declare updatedAt: DateTime | null } export class GroupSchema extends BaseModel { - static $columns = ['active', 'createdAt', 'id', 'lastIndexedId', 'name', 'updatedAt'] as const + static $columns = ['active', 'createdAt', 'fetchBodies', 'id', 'lastIndexedId', 'lastQueuedId', 'name', 'updatedAt'] as const $columns = GroupSchema.$columns @column() declare active: boolean @column.dateTime({ autoCreate: true }) declare createdAt: DateTime | null + @column() + declare fetchBodies: boolean @column({ isPrimary: true }) declare id: number @column() declare lastIndexedId: bigint | number | null @column() + declare lastQueuedId: bigint | number + @column() declare name: string @column.dateTime({ autoCreate: true, autoUpdate: true }) declare updatedAt: DateTime | null } +export class PartConflictSchema extends BaseModel { + static $columns = ['articleNumber', 'binaryId', 'bytes', 'fileCrc32', 'md516K', 'messageId', 'number', 'partSize', 'pcrc32'] as const + $columns = PartConflictSchema.$columns + @column() + declare articleNumber: bigint | number | null + @column({ isPrimary: true }) + declare binaryId: bigint | number + @column() + declare bytes: number + @column() + declare fileCrc32: string | null + @column() + declare md516K: string | null + @column() + declare messageId: string + @column() + declare number: number + @column() + declare partSize: number | null + @column() + declare pcrc32: string | null +} + +export class PartSchema extends BaseModel { + static $columns = ['articleNumber', 'binaryId', 'bytes', 'fileCrc32', 'md516K', 'messageId', 'number', 'partSize', 'pcrc32'] as const + $columns = PartSchema.$columns + @column() + declare articleNumber: bigint | number | null + @column({ isPrimary: true }) + declare binaryId: bigint | number + @column() + declare bytes: number + @column() + declare fileCrc32: string | null + @column() + declare md516K: string | null + @column() + declare messageId: string + @column() + declare number: number + @column() + declare partSize: number | null + @column() + declare pcrc32: string | null +} + +export class ReleaseFileSchema extends BaseModel { + static $columns = ['md5', 'md516K', 'name', 'par2FileId', 'releaseId', 'setId', 'size'] as const + $columns = ReleaseFileSchema.$columns + @column() + declare md5: string | null + @column() + declare md516K: string + @column() + declare name: string + @column() + declare par2FileId: string + @column({ isPrimary: true }) + declare releaseId: bigint | number + @column() + declare setId: string | null + @column() + declare size: bigint | number +} + +export class ReleaseSchema extends BaseModel { + static $columns = ['category', 'complete', 'container', 'contents', 'encrypted', 'fileCount', 'fileTotal', 'firstSeenAt', 'groups', 'id', 'lastSeenAt', 'name', 'password', 'postedAt', 'poster', 'releaseKey', 'size', 'source'] as const + $columns = ReleaseSchema.$columns + @column() + declare category: number | null + @column() + declare complete: boolean + @column() + declare container: string | null + @column() + declare contents: any | null + @column() + declare encrypted: boolean | null + @column() + declare fileCount: number + @column() + declare fileTotal: number | null + @column.dateTime() + declare firstSeenAt: DateTime + @column() + declare groups: any + @column({ isPrimary: true }) + declare id: bigint | number + @column.dateTime() + declare lastSeenAt: DateTime + @column() + declare name: string + @column({ serializeAs: null }) + declare password: string | null + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string | null + @column() + declare releaseKey: string + @column() + declare size: bigint | number + @column() + declare source: string | null +} + +export class SpotSchema extends BaseModel { + static $columns = ['articleNumber', 'category', 'createdAt', 'error', 'groupName', 'keyId', 'messageId', 'postedAt', 'poster', 'processedAt', 'releaseId', 'size', 'spotterId', 'status', 'subcats', 'tag', 'title', 'website'] as const + $columns = SpotSchema.$columns + @column() + declare articleNumber: bigint | number | null + @column() + declare category: number | null + @column.dateTime({ autoCreate: true }) + declare createdAt: DateTime + @column() + declare error: string | null + @column() + declare groupName: string + @column() + declare keyId: number | null + @column({ isPrimary: true }) + declare messageId: string + @column.dateTime() + declare postedAt: DateTime | null + @column() + declare poster: string + @column.dateTime() + declare processedAt: DateTime | null + @column() + declare releaseId: bigint | number | null + @column() + declare size: bigint | number | null + @column() + declare spotterId: string | null + @column() + declare status: string + @column() + declare subcats: any | null + @column() + declare tag: string | null + @column() + declare title: string + @column() + declare website: string | null +} + export class UserSchema extends BaseModel { static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const $columns = UserSchema.$columns diff --git a/lib/content/index.ts b/lib/content/index.ts new file mode 100644 index 0000000..84dc81a --- /dev/null +++ b/lib/content/index.ts @@ -0,0 +1,4 @@ +/** + * Content recognition by magic bytes, independent of AdonisJS. + */ +export { sniffContentType, type ContentType } from './sniff.js' diff --git a/lib/content/sniff.ts b/lib/content/sniff.ts new file mode 100644 index 0000000..63aec85 --- /dev/null +++ b/lib/content/sniff.ts @@ -0,0 +1,72 @@ +/** + * Recognises a file type from its first bytes, independent of AdonisJS. + * `nzbe` is an encrypted-NZB container seen on Usenet (LOG.md 2026-09-28 (11)): + * "nzbe", 16 random bytes (salt/IV), then ciphertext. + */ +export type ContentType = + | 'nzbe' + | 'par2' + | 'rar4' + | 'rar5' + | '7z' + | 'zip' + | 'zstd' + | 'gzip' + | 'xz' + | 'matroska' + | 'mp4' + | 'rclone' + | 'iso' + | 'udf' + | 'pdf' + | 'nzb' + | 'jpeg' + | 'png' + | 'mp3' + | 'flac' + | 'avi' + | 'wav' + | 'ogg' + | 'bzip2' + +const SIGNATURES: [ContentType, number, number[]][] = [ + ['nzbe', 0, [0x6e, 0x7a, 0x62, 0x65]], + ['par2', 0, [0x50, 0x41, 0x52, 0x32, 0x00, 0x50, 0x4b, 0x54]], + ['rar5', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]], + ['rar4', 0, [0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]], + ['7z', 0, [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c]], + ['zip', 0, [0x50, 0x4b, 0x03, 0x04]], + ['zstd', 0, [0x28, 0xb5, 0x2f, 0xfd]], + ['gzip', 0, [0x1f, 0x8b]], + ['xz', 0, [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00]], + ['matroska', 0, [0x1a, 0x45, 0xdf, 0xa3]], + ['mp4', 4, [0x66, 0x74, 0x79, 0x70]], // "ftyp" box + ['rclone', 0, [0x52, 0x43, 0x4c, 0x4f, 0x4e, 0x45]], // rclone crypt + ['pdf', 0, [0x25, 0x50, 0x44, 0x46, 0x2d]], + ['jpeg', 0, [0xff, 0xd8, 0xff]], + ['png', 0, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]], + ['mp3', 0, [0x49, 0x44, 0x33]], // "ID3" tag + ['flac', 0, [0x66, 0x4c, 0x61, 0x43]], // "fLaC" + ['ogg', 0, [0x4f, 0x67, 0x67, 0x53]], // "OggS" + ['bzip2', 0, [0x42, 0x5a, 0x68]], // "BZh" +] + +export function sniffContentType(data: Buffer): ContentType | null { + for (const [type, offset, bytes] of SIGNATURES) { + if (data.length >= offset + bytes.length && bytes.every((b, i) => data[offset + i] === b)) return type + } + // Disc images, visible when the first part is big enough (sector 16 at 32 KiB): + // ISO 9660 has "CD001"; UDF (DVD/Blu-ray) has BEA01 followed by NSR02/NSR03. + if (data.length >= 0x8006 && data.subarray(0x8001, 0x8006).toString('latin1') === 'CD001') return 'iso' + if (data.length >= 0x8806 && data.subarray(0x8001, 0x8006).toString('latin1') === 'BEA01' && + /^NSR0[23]$/.test(data.subarray(0x8801, 0x8806).toString('latin1'))) return 'udf' + // RIFF container: AVI or WAV by form type. + if (data.length >= 12 && data.subarray(0, 4).toString('latin1') === 'RIFF') { + const form = data.subarray(8, 12).toString('latin1') + if (form === 'AVI ') return 'avi' + if (form === 'WAVE') return 'wav' + } + const head = data.subarray(0, 512).toString('latin1') + if (/^\s*(<\?xml[^>]*>\s*)?(])/.test(head)) return 'nzb' + return null +} diff --git a/lib/newznab/categories.ts b/lib/newznab/categories.ts new file mode 100644 index 0000000..35a4bfd --- /dev/null +++ b/lib/newznab/categories.ts @@ -0,0 +1,40 @@ +/** Newznab categories this indexer assigns (see release_category() in SQL). */ +export const CATEGORIES: { id: number; name: string; subcats: { id: number; name: string }[] }[] = [ + { id: 2000, name: 'Movies', subcats: [{ id: 2030, name: 'SD' }, { id: 2040, name: 'HD' }, { id: 2045, name: 'UHD' }] }, + { id: 3000, name: 'Audio', subcats: [{ id: 3010, name: 'MP3' }, { id: 3040, name: 'Lossless' }] }, + { id: 4000, name: 'PC', subcats: [] }, + { id: 5000, name: 'TV', subcats: [{ id: 5030, name: 'SD' }, { id: 5040, name: 'HD' }, { id: 5045, name: 'UHD' }] }, + { id: 7000, name: 'Books', subcats: [{ id: 7020, name: 'Ebook' }] }, + { id: 8000, name: 'Other', subcats: [{ id: 8010, name: 'Misc' }] }, +] + +/** Parent category of a (sub)category, e.g. 5040 → 5000. */ +export function parentCategory(id: number): number { + return Math.floor(id / 1000) * 1000 +} + +/** "TV > HD" style label. */ +export function categoryLabel(id: number): string { + const parent = CATEGORIES.find((c) => c.id === parentCategory(id)) + if (!parent) return 'Other' + const sub = parent.subcats.find((s) => s.id === id) + return sub ? `${parent.name} > ${sub.name}` : parent.name +} + +/** + * Expands requested categories (`cat=5000,2040`) to the concrete ones to + * match: a parent includes all of its subcategories. + */ +export function expandCategories(requested: number[]): number[] { + const result = new Set() + for (const id of requested) { + const parent = CATEGORIES.find((c) => c.id === id) + if (parent) { + result.add(parent.id) + for (const sub of parent.subcats) result.add(sub.id) + } else { + result.add(id) + } + } + return [...result] +} diff --git a/lib/newznab/index.ts b/lib/newznab/index.ts new file mode 100644 index 0000000..632c0a9 --- /dev/null +++ b/lib/newznab/index.ts @@ -0,0 +1,5 @@ +/** + * Newznab API formatting, independent of AdonisJS. See CONCEPT.md (D23). + */ +export { CATEGORIES, categoryLabel, expandCategories, parentCategory } from './categories.js' +export { capsXml, errorXml, resultsXml, type NewznabItem, type NewznabServer } from './xml.js' diff --git a/lib/newznab/xml.ts b/lib/newznab/xml.ts new file mode 100644 index 0000000..22ee71d --- /dev/null +++ b/lib/newznab/xml.ts @@ -0,0 +1,95 @@ +import { create } from 'xmlbuilder2' +import { CATEGORIES, categoryLabel, parentCategory } from './categories.js' + +const NEWZNAB_NS = 'http://www.newznab.com/DTD/2010/feeds/attributes/' + +export interface NewznabItem { + id: number + title: string + category: number + size: number + files: number + postedAt: Date | null + poster: string | null + groups: string[] + password: boolean + season?: string + episode?: string +} + +export interface NewznabServer { + title: string + /** Absolute base URL, e.g. http://homelab:3333 */ + baseUrl: string + /** Appended to download links so a client can fetch without extra config. */ + apiKey?: string +} + +/** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */ +export function errorXml(code: number, description: string): string { + return create({ version: '1.0', encoding: 'UTF-8' }) + .ele('error', { code: String(code), description }) + .end({ prettyPrint: true }) +} + +export function capsXml(server: NewznabServer, limits: { max: number; default: number }): string { + const caps = create({ version: '1.0', encoding: 'UTF-8' }).ele('caps') + caps.ele('server', { version: '1.0', title: server.title, strapline: 'Usenet indexer', url: `${server.baseUrl}/` }) + caps.ele('limits', { max: String(limits.max), default: String(limits.default) }) + caps.ele('registration', { available: 'no', open: 'no' }) + const searching = caps.ele('searching') + searching.ele('search', { available: 'yes', supportedParams: 'q' }) + searching.ele('tv-search', { available: 'yes', supportedParams: 'q,season,ep' }) + searching.ele('movie-search', { available: 'yes', supportedParams: 'q' }) + searching.ele('audio-search', { available: 'yes', supportedParams: 'q' }) + searching.ele('book-search', { available: 'yes', supportedParams: 'q' }) + const categories = caps.ele('categories') + for (const category of CATEGORIES) { + const element = categories.ele('category', { id: String(category.id), name: category.name }) + for (const sub of category.subcats) element.ele('subcat', { id: String(sub.id), name: sub.name }) + } + return caps.end({ prettyPrint: true }) +} + +export function resultsXml(server: NewznabServer, items: NewznabItem[], offset: number, total: number): string { + const rss = create({ version: '1.0', encoding: 'UTF-8' }).ele('rss', { + 'version': '2.0', + 'xmlns:atom': 'http://www.w3.org/2005/Atom', + 'xmlns:newznab': NEWZNAB_NS, + }) + const channel = rss.ele('channel') + channel.ele('title').txt(server.title) + channel.ele('description').txt(`${server.title} API results`) + channel.ele('link').txt(`${server.baseUrl}/`) + channel.ele('newznab:response', { offset: String(offset), total: String(total) }) + + const key = server.apiKey ? `&apikey=${encodeURIComponent(server.apiKey)}` : '' + for (const item of items) { + const getUrl = `${server.baseUrl}/api?t=get&id=${item.id}${key}` + const pubDate = (item.postedAt ?? new Date()).toUTCString() + const element = channel.ele('item') + element.ele('title').txt(item.title) + element.ele('guid', { isPermaLink: 'false' }).txt(String(item.id)) + element.ele('link').txt(getUrl) + element.ele('comments').txt(`${server.baseUrl}/?release=${item.id}`) + element.ele('pubDate').txt(pubDate) + element.ele('category').txt(categoryLabel(item.category)) + element.ele('description').txt(item.title) + element.ele('enclosure', { url: getUrl, length: String(item.size), type: 'application/x-nzb' }) + + const attr = (name: string, value: string | number) => + element.ele('newznab:attr', { name, value: String(value) }) + attr('category', parentCategory(item.category)) + if (item.category !== parentCategory(item.category)) attr('category', item.category) + attr('size', item.size) + attr('files', item.files) + attr('grabs', 0) + attr('password', item.password ? 1 : 0) + if (item.poster) attr('poster', item.poster) + if (item.groups.length) attr('group', item.groups[0]) + attr('usenetdate', pubDate) + if (item.season) attr('season', item.season) + if (item.episode) attr('episode', item.episode) + } + return rss.end({ prettyPrint: true }) +} diff --git a/lib/nntp/group.ts b/lib/nntp/group.ts new file mode 100644 index 0000000..24785c8 --- /dev/null +++ b/lib/nntp/group.ts @@ -0,0 +1,23 @@ +export interface GroupInfo { + count: bigint + first: bigint + last: bigint + name: string +} + +/** + * Parses a `211 count first last name` GROUP reply (RFC 3977 §6.1.1). + * + * nntp-js's own `group()` destructures this reply without skipping the status + * code, so its `first`/`last` are off by one field. Pass it `result.resp`. + */ +export function parseGroupResponse(resp: string): GroupInfo { + const match = /^211\s+(\d+)\s+(\d+)\s+(\d+)\s+(\S+)/.exec(resp.trim()) + if (!match) throw new Error(`Unexpected GROUP response: ${resp}`) + return { + count: BigInt(match[1]), + first: BigInt(match[2]), + last: BigInt(match[3]), + name: match[4], + } +} diff --git a/lib/nntp/xref.ts b/lib/nntp/xref.ts new file mode 100644 index 0000000..2a95dd2 --- /dev/null +++ b/lib/nntp/xref.ts @@ -0,0 +1,13 @@ +/** + * Returns the group names from an Xref header, + * e.g. `news.eweka.nl alt.binaries.test:123 alt.binaries.x:456`. + */ +export function parseXrefGroups(xref: string | undefined): string[] { + if (!xref) return [] + return xref + .trim() + .split(/\s+/) + .slice(1) + .map((entry) => entry.slice(0, entry.lastIndexOf(':'))) + .filter(Boolean) +} diff --git a/lib/nzb/build.ts b/lib/nzb/build.ts new file mode 100644 index 0000000..f3ccabc --- /dev/null +++ b/lib/nzb/build.ts @@ -0,0 +1,54 @@ +import { create } from 'xmlbuilder2' + +export interface NzbFile { + /** Name the downloader should use; the real name when it is known. */ + name: string + poster: string + /** Posting time in milliseconds. */ + date: number + groups: string[] + /** Segment number → message-id (without angle brackets) and article size. */ + segments: Record +} + +export interface NzbOptions { + /** Shown by downloaders as the job name. */ + title?: string + /** Archive password, when known. */ + password?: string +} + +/** + * Builds an NZB 1.1 document (newzbin DTD). The subject follows the common + * `"name" yEnc (1/N)` form, which downloaders use to name the file. + */ +export function buildNzb(files: NzbFile[], options: NzbOptions = {}): string { + const root = create({ version: '1.0', encoding: 'UTF-8' }) + .dtd({ pubID: '-//newzBin//DTD NZB 1.1//EN', sysID: 'http://www.newzbin.com/DTD/nzb/nzb-1.1.dtd' }) + .ele('nzb', { xmlns: 'http://www.newzbin.com/DTD/2003/nzb' }) + + if (options.title || options.password) { + const head = root.ele('head') + if (options.title) head.ele('meta', { type: 'name' }).txt(options.title) + if (options.password) head.ele('meta', { type: 'password' }).txt(options.password) + } + + for (const file of files) { + const numbers = Object.keys(file.segments) + .map(Number) + .sort((a, b) => a - b) + const element = root.ele('file', { + poster: file.poster, + date: String(Math.floor(file.date / 1000)), + subject: `"${file.name}" yEnc (1/${numbers.length})`, + }) + const groups = element.ele('groups') + for (const group of file.groups) groups.ele('group').txt(group) + const segments = element.ele('segments') + for (const number of numbers) { + const segment = file.segments[String(number)] + segments.ele('segment', { bytes: String(segment.size), number: String(number) }).txt(segment.id) + } + } + return root.end({ prettyPrint: true }) +} diff --git a/lib/nzb/index.ts b/lib/nzb/index.ts new file mode 100644 index 0000000..20f7090 --- /dev/null +++ b/lib/nzb/index.ts @@ -0,0 +1,5 @@ +/** + * NZB generation, independent of AdonisJS. + */ +export { buildNzb, type NzbFile, type NzbOptions } from './build.js' +export { parseNzb, type ParsedNzb, type ParsedNzbFile } from './parse.js' diff --git a/lib/nzb/parse.ts b/lib/nzb/parse.ts new file mode 100644 index 0000000..fec1fec --- /dev/null +++ b/lib/nzb/parse.ts @@ -0,0 +1,72 @@ +export interface ParsedNzbFile { + poster: string + /** Posting time in seconds (as in the NZB). */ + date: number + subject: string + groups: string[] + segments: { number: number; bytes: number; id: string }[] +} + +export interface ParsedNzb { + /** values, e.g. name, password, category. */ + meta: Record + files: ParsedNzbFile[] +} + +const ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" } + +function decode(text: string): string { + return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => { + if (entity[0] === '#') { + const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10) + return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match + } + return ENTITIES[entity.toLowerCase()] ?? match + }) +} + +function attributes(tag: string): Record { + const result: Record = {} + for (const m of tag.matchAll(/([\w:-]+)\s*=\s*("([^"]*)"|'([^']*)')/g)) result[m[1]] = decode(m[3] ?? m[4] ?? '') + return result +} + +/** + * Parses an NZB (1.0/1.1 and the older 0.9 DTD). A tolerant scanner rather + * than a full XML parser: NZBs are flat, and this never resolves DTDs or + * external entities. Returns null when there is no root. + */ +export function parseNzb(xml: string, limits = { maxFiles: 20_000 }): ParsedNzb | null { + if (!/]/i.test(xml)) return null + + const meta: Record = {} + const head = /([\s\S]*?)<\/head>/i.exec(xml)?.[1] ?? '' + for (const m of head.matchAll(/]*)>([\s\S]*?)<\/meta>/gi)) { + const type = attributes(m[1]).type + if (type) meta[type.toLowerCase()] = decode(m[2].trim()) + } + + const files: ParsedNzbFile[] = [] + for (const m of xml.matchAll(/]*)>([\s\S]*?)<\/file>/gi)) { + if (files.length >= limits.maxFiles) break + const attrs = attributes(m[1]) + const body = m[2] + const groups = [...body.matchAll(/([^<]*)<\/group>/gi)].map((g) => decode(g[1].trim())).filter(Boolean) + const segments = [...body.matchAll(/]*)>([^<]*)<\/segment>/gi)] + .map((s) => { + const a = attributes(s[1]) + return { number: Number(a.number), bytes: Number(a.bytes) || 0, id: decode(s[2].trim()).replace(/^<|>$/g, '') } + }) + .filter((s) => Number.isInteger(s.number) && s.number > 0 && s.id) + .sort((x, y) => x.number - y.number) + if (segments.length === 0) continue + files.push({ + poster: attrs.poster ?? '', + date: Number(attrs.date) || 0, + subject: attrs.subject ?? '', + groups, + segments, + }) + } + return { meta, files } +} diff --git a/lib/par2/index.ts b/lib/par2/index.ts new file mode 100644 index 0000000..37cb73f --- /dev/null +++ b/lib/par2/index.ts @@ -0,0 +1,5 @@ +/** + * PAR2 parsing, independent of AdonisJS. Used to recover the real filenames + * and sizes of obfuscated posts. See CONCEPT.md (D16). + */ +export { parsePar2, isPar2, type Par2File, type Par2Set } from './parse.js' diff --git a/lib/par2/parse.ts b/lib/par2/parse.ts new file mode 100644 index 0000000..6a30e79 --- /dev/null +++ b/lib/par2/parse.ts @@ -0,0 +1,74 @@ +import { createHash } from 'node:crypto' + +export interface Par2File { + /** File ID (hex): MD5 of md5_16k, length and name, as defined by PAR2. */ + fileId: string + /** MD5 of the whole file (hex). */ + md5: string + /** MD5 of the first 16 KiB (hex). */ + md5_16k: string + length: number + name: string +} + +export interface Par2Set { + /** Recovery set ID (hex), shared by every packet of one par2 set. */ + recoverySetId: string + files: Par2File[] +} + +const MAGIC = Buffer.from('PAR2\0PKT', 'latin1') +const HEADER_SIZE = 64 +const FILE_DESC = Buffer.from('PAR 2.0\0FileDesc', 'latin1') + +export function isPar2(data: Buffer): boolean { + return data.subarray(0, MAGIC.length).equals(MAGIC) +} + +/** + * Reads the FileDesc packets of a par2 file (index or recovery volume). + * Packets with a bad length or checksum are skipped; parsing resumes at the + * next magic sequence. Returns null when no valid FileDesc packet is found. + */ +export function parsePar2(data: Buffer): Par2Set | null { + const files = new Map() + let recoverySetId: string | undefined + let offset = data.indexOf(MAGIC) + + while (offset !== -1 && offset + HEADER_SIZE <= data.length) { + const length = Number(data.readBigUInt64LE(offset + 8)) + const valid = + length >= HEADER_SIZE && + length % 4 === 0 && + offset + length <= data.length && + createHash('md5') + .update(data.subarray(offset + 32, offset + length)) + .digest() + .equals(data.subarray(offset + 16, offset + 32)) + + if (!valid) { + offset = data.indexOf(MAGIC, offset + 1) + continue + } + + const setId = data.subarray(offset + 32, offset + 48).toString('hex') + const type = data.subarray(offset + 48, offset + 64) + if (type.equals(FILE_DESC) && (!recoverySetId || recoverySetId === setId)) { + recoverySetId = setId + const body = data.subarray(offset + HEADER_SIZE, offset + length) + const fileId = body.subarray(0, 16).toString('hex') + files.set(fileId, { + fileId, + md5: body.subarray(16, 32).toString('hex'), + md5_16k: body.subarray(32, 48).toString('hex'), + length: Number(body.readBigUInt64LE(48)), + // The name is padded with NULs to a multiple of 4 bytes. + name: body.subarray(56).toString('utf8').replace(/\0+$/, ''), + }) + } + offset += length + } + + if (!recoverySetId || files.size === 0) return null + return { recoverySetId, files: [...files.values()] } +} diff --git a/lib/rar/index.ts b/lib/rar/index.ts new file mode 100644 index 0000000..5faa2e6 --- /dev/null +++ b/lib/rar/index.ts @@ -0,0 +1,5 @@ +/** + * RAR (4.x and 5.0) header parsing, independent of AdonisJS. Lists archive + * contents from the first segment of a first volume. See CONCEPT.md (D19). + */ +export { parseRarHeaders, rarFormat, decodeRar4Name, storedEntryData, type RarInfo, type RarEntry } from './parse.js' diff --git a/lib/rar/parse.ts b/lib/rar/parse.ts new file mode 100644 index 0000000..0b23e3a --- /dev/null +++ b/lib/rar/parse.ts @@ -0,0 +1,333 @@ +import { crc32 } from 'node:zlib' + +export interface RarEntry { + name: string + /** Unpacked size in bytes (null when the archiver did not record it). */ + size: number | null + packedSize: number + directory: boolean + /** The file's data is password protected. */ + encrypted: boolean + /** Continues from the previous volume. */ + splitBefore: boolean + /** Continues in the next volume. */ + splitAfter: boolean + /** Stored without compression, so the data can be copied out as is. */ + stored: boolean + /** Offset of the entry's (packed) data in the parsed buffer. */ + dataOffset: number +} + +export interface RarInfo { + format: 'rar4' | 'rar5' + /** Part of a multi-volume set. */ + volume: boolean + /** First volume of the set (null when the format version does not say). */ + firstVolume: boolean | null + solid: boolean + /** File names are encrypted too; nothing can be listed without the password. */ + headersEncrypted: boolean + entries: RarEntry[] + /** The buffer ended before the end-of-archive header (normal for one segment). */ + truncated: boolean +} + +const RAR4_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]) +const RAR5_MARKER = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]) + +export function rarFormat(data: Buffer): 'rar4' | 'rar5' | null { + if (data.subarray(0, RAR5_MARKER.length).equals(RAR5_MARKER)) return 'rar5' + if (data.subarray(0, RAR4_MARKER.length).equals(RAR4_MARKER)) return 'rar4' + return null +} + +/** + * Lists a RAR archive from its headers only, so the first segment of a first + * volume is enough. Returns null when the data is not a RAR archive. + */ +export function parseRarHeaders(data: Buffer): RarInfo | null { + const format = rarFormat(data) + if (format === 'rar4') return parseRar4(data) + if (format === 'rar5') return parseRar5(data) + return null +} + +// --- RAR 4.x (RAR 1.5 – 4.x archive format) ------------------------------- + +const R4_MAIN = 0x73 +const R4_FILE = 0x74 +const R4_END = 0x7b +const R4_LONG_BLOCK = 0x8000 + +function parseRar4(data: Buffer): RarInfo { + const info: RarInfo = { + format: 'rar4', + volume: false, + firstVolume: null, + solid: false, + headersEncrypted: false, + entries: [], + truncated: true, + } + let p = RAR4_MARKER.length + + while (p + 7 <= data.length) { + const headCrc = data.readUInt16LE(p) + const type = data[p + 2] + const flags = data.readUInt16LE(p + 3) + const size = data.readUInt16LE(p + 5) + if (size < 7 || p + size > data.length) break + + if (type === R4_END) { + info.truncated = false + break + } + + let next = p + size + (flags & R4_LONG_BLOCK && size >= 11 ? data.readUInt32LE(p + 7) : 0) + + if (type === R4_MAIN) { + info.volume = Boolean(flags & 0x0001) + info.solid = Boolean(flags & 0x0008) + info.firstVolume = info.volume ? Boolean(flags & 0x0100) : true + if (flags & 0x0080) { + info.headersEncrypted = true + break + } + } else if (type === R4_FILE) { + if (size < 32 || (crc32(data.subarray(p + 2, p + size)) & 0xffff) !== headCrc) break + const large = Boolean(flags & 0x0100) + let packed = data.readUInt32LE(p + 7) + let unpacked = data.readUInt32LE(p + 11) + const nameSize = data.readUInt16LE(p + 26) + let nameStart = p + 32 + if (large) { + packed += data.readUInt32LE(p + 32) * 2 ** 32 + unpacked += data.readUInt32LE(p + 36) * 2 ** 32 + nameStart = p + 40 + } + const rawName = data.subarray(nameStart, nameStart + nameSize) + info.entries.push({ + name: flags & 0x0200 ? decodeRar4Name(rawName) : rawName.toString('latin1'), + // 0xffffffff with LHD_LARGE unset means "unknown" in some archivers. + size: unpacked === 0xffffffff && !large ? null : unpacked, + packedSize: packed, + directory: (flags & 0x00e0) === 0x00e0, + encrypted: Boolean(flags & 0x0004), + splitBefore: Boolean(flags & 0x0001), + splitAfter: Boolean(flags & 0x0002), + stored: data[p + 25] === 0x30, + dataOffset: p + size, + }) + next = p + size + packed + } + + if (next <= p) break + p = next + } + // Archivers before RAR 3.0 don't set the first-volume flag; a first + // volume is one whose first file does not continue from a previous one. + if (info.volume && !info.firstVolume && info.entries.length > 0 && !info.entries[0].splitBefore) { + info.firstVolume = true + } + return info +} + +/** + * RAR 4 stores Unicode names as "ASCII\0", where the + * compressed part refers back to the ASCII bytes (unrar's DecodeFileName). + */ +export function decodeRar4Name(raw: Buffer): string { + const zero = raw.indexOf(0) + if (zero === -1) return raw.toString('utf8') + const ascii = raw.subarray(0, zero) + const enc = raw.subarray(zero + 1) + if (enc.length === 0) return ascii.toString('latin1') + + const out: number[] = [] + const highByte = enc[0] + let encPos = 1 + let flags = 0 + let flagBits = 0 + while (encPos < enc.length) { + if (flagBits === 0) { + flags = enc[encPos++] + flagBits = 8 + } + switch (flags >> 6) { + case 0: + out.push(enc[encPos++]) + break + case 1: + out.push(enc[encPos++] + (highByte << 8)) + break + case 2: + out.push(enc[encPos] + (enc[encPos + 1] << 8)) + encPos += 2 + break + case 3: { + let length = enc[encPos++] + if (length & 0x80) { + const correction = enc[encPos++] + for (length = (length & 0x7f) + 2; length > 0 && out.length < ascii.length; length--) { + out.push(((ascii[out.length] + correction) & 0xff) + (highByte << 8)) + } + } else { + for (length += 2; length > 0 && out.length < ascii.length; length--) { + out.push(ascii[out.length]) + } + } + break + } + } + flags = (flags << 2) & 0xff + flagBits -= 2 + } + return String.fromCharCode(...out) +} + +// --- RAR 5.0 ---------------------------------------------------------------- + +const R5_MAIN = 1 +const R5_FILE = 2 +const R5_ENCRYPTION = 4 +const R5_END = 5 + +/** Reads a RAR5 variable-length integer: 7 bits per byte, little-endian. */ +function vint(data: Buffer, offset: number): { value: number; next: number } | null { + let value = 0 + let shift = 1 + for (let i = offset; i < data.length && i < offset + 10; i++) { + value += (data[i] & 0x7f) * shift + shift *= 128 + if (!(data[i] & 0x80)) return { value, next: i + 1 } + } + return null +} + +function parseRar5(data: Buffer): RarInfo { + const info: RarInfo = { + format: 'rar5', + volume: false, + firstVolume: null, + solid: false, + headersEncrypted: false, + entries: [], + truncated: true, + } + let p = RAR5_MARKER.length + + while (p + 5 <= data.length) { + const headCrc = data.readUInt32LE(p) + const headerSize = vint(data, p + 4) + if (!headerSize || headerSize.value === 0) break + const headerEnd = headerSize.next + headerSize.value + if (headerEnd > data.length) break + if ((crc32(data.subarray(p + 4, headerEnd)) >>> 0) !== headCrc) break + + const type = vint(data, headerSize.next) + const flags = type && vint(data, type.next) + if (!type || !flags) break + let cursor = flags.next + let extraSize = 0 + let dataSize = 0 + if (flags.value & 0x0001) { + const v = vint(data, cursor) + if (!v) break + extraSize = v.value + cursor = v.next + } + if (flags.value & 0x0002) { + const v = vint(data, cursor) + if (!v) break + dataSize = v.value + cursor = v.next + } + + if (type.value === R5_END) { + info.truncated = false + break + } + if (type.value === R5_ENCRYPTION) { + info.headersEncrypted = true + break + } + + if (type.value === R5_MAIN) { + const archiveFlags = vint(data, cursor) + if (!archiveFlags) break + info.volume = Boolean(archiveFlags.value & 0x0001) + // A volume number is only stored for volumes after the first. + info.firstVolume = info.volume ? !(archiveFlags.value & 0x0002) : true + info.solid = Boolean(archiveFlags.value & 0x0004) + } else if (type.value === R5_FILE) { + const entry = parseRar5File(data, cursor, headerEnd - extraSize, headerEnd, flags.value, dataSize) + if (!entry) break + info.entries.push(entry) + } + + const next = headerEnd + dataSize + if (next <= p) break + p = next + } + return info +} + +function parseRar5File( + data: Buffer, + start: number, + extraStart: number, + headerEnd: number, + headerFlags: number, + dataSize: number +): RarEntry | null { + const fileFlags = vint(data, start) + const unpacked = fileFlags && vint(data, fileFlags.next) + const attributes = unpacked && vint(data, unpacked.next) + if (!fileFlags || !unpacked || !attributes) return null + let cursor = attributes.next + if (fileFlags.value & 0x0002) cursor += 4 // mtime + if (fileFlags.value & 0x0004) cursor += 4 // data CRC32 + const compression = vint(data, cursor) + const hostOs = compression && vint(data, compression.next) + const nameLength = hostOs && vint(data, hostOs.next) + if (!compression || !hostOs || !nameLength) return null + const name = data.subarray(nameLength.next, nameLength.next + nameLength.value).toString('utf8') + // Compression information: bits 7-9 are the method, 0 is "store". + const method = Math.floor(compression.value / 128) & 0x7 + + // Extra area records: size, type, data. Type 1 is file encryption. + let encrypted = false + let e = extraStart + while (e < headerEnd) { + const recordSize = vint(data, e) + const recordType = recordSize && vint(data, recordSize.next) + if (!recordSize || !recordType || recordSize.value === 0) break + if (recordType.value === 0x01) encrypted = true + e = recordSize.next + recordSize.value + } + + return { + name, + size: fileFlags.value & 0x0008 ? null : unpacked.value, + packedSize: dataSize, + directory: Boolean(fileFlags.value & 0x0001), + encrypted, + splitBefore: Boolean(headerFlags & 0x0008), + splitAfter: Boolean(headerFlags & 0x0010), + stored: method === 0, + dataOffset: headerEnd, + } +} + +/** + * The data of an entry stored without compression, copied out of the + * archive buffer. Null when it is compressed, encrypted, split over volumes + * or not completely in the buffer. + */ +export function storedEntryData(data: Buffer, entry: RarEntry): Buffer | null { + if (!entry.stored || entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null + if (entry.size === null || entry.size !== entry.packedSize) return null + const end = entry.dataOffset + entry.packedSize + if (end > data.length) return null + return data.subarray(entry.dataOffset, end) +} diff --git a/lib/release/index.ts b/lib/release/index.ts new file mode 100644 index 0000000..e4ede7b --- /dev/null +++ b/lib/release/index.ts @@ -0,0 +1,4 @@ +/** + * Grouping of files into releases (downloads), independent of AdonisJS. + */ +export { releaseIdentity, par2ReleaseIdentity, par2ReleaseName, rarReleaseIdentity, fileStem, subjectRemainder, namesRelease, isPostToken, type ReleaseIdentity, type ReleaseInput } from './release_key.js' diff --git a/lib/release/release_key.ts b/lib/release/release_key.ts new file mode 100644 index 0000000..17a2074 --- /dev/null +++ b/lib/release/release_key.ts @@ -0,0 +1,211 @@ +import { createHash } from 'node:crypto' +import { looksLikeFilename } from '../subject/file_key.js' + +export interface ReleaseIdentity { + key: string + /** Human-readable name: the subject text or the shared filename stem. */ + name: string + /** Number of files announced by the `[i/M]` counter, when present. */ + fileTotal?: number + /** Which rule recognised the release. */ + source: 'subject' | 'post' | 'stem' | 'weak-stem' | 'par2' | 'rar' | 'nzb' | 'spot' +} + +export interface ReleaseInput { + poster: string + /** Subject without segment counter (the binary's `subject`). */ + subject: string + /** Filename of the binary. */ + name: string + /** True when the identity came from the yEnc body (obfuscated post). */ + weak: boolean +} + +// Suffixes that differ between the files of one release. Stripped repeatedly, +// so "x.mkv.vol03+04.par2" and "x.part07.rar" both reduce to "x". +const SUFFIXES = [ + /\.vol\d+[+-]\d+\.par2$/i, + /\.par2$/i, + /\.part\d+\.rar$/i, + /\.r\d{2,3}$/i, + /\.rar$/i, + /\.7z(\.\d{3})?$/i, + /\.zip(\.\d{3})?$/i, + // Split volumes (x.mkv.001), not codec names (H.264, H.265). + /(?<=\.(7z|zip|rar|tar|mkv|mp4|avi|m4v|ts|m2ts|iso|img|bin|vob))\.\d{3}$/i, + /\.(nfo|sfv|nzb|srr|srs|txt|jpg|png|md5|sha1|srt|sub|idx|ass|ssa|m3u|m3u8|cue|log)$/i, + /\.tar$/i, + /\.(zst|gz|xz|bz2)$/i, + /\.(mkv|mp4|avi|m4v|ts|iso|img|bin|flac|mp3|epub|pdf)$/i, +] + +/** Strips per-file suffixes (volume numbers, archive and par2 extensions) from a filename. */ +export function fileStem(name: string): string { + let stem = name.trim() + let changed = true + while (changed) { + changed = false + for (const suffix of SUFFIXES) { + const next = stem.replace(suffix, '') + if (next !== stem && next.length > 0) { + stem = next + changed = true + } + } + } + return stem +} + +/** + * What remains of a subject once the filename, counters, "yEnc" and sizes are + * removed. Posters that name the release outside the quotes put it here. + */ +export function subjectRemainder(subject: string): string { + return subject + .replace(/"[^"]*"/g, ' ') + .replace(/\[\d+\/\d+\]|\(\d+\/\d+\)/g, ' ') + // File sizes follow "yEnc" at the end; other numbers (years, post ids) stay. + .replace(/\byEnc\b(\s+\d+)?\s*$/i, ' ') + .replace(/\byEnc\b/gi, ' ') + .replace(/[\s\-–.:|]+/g, ' ') + .trim() +} + +/** + * Posting tools put random per-file tokens and poster IDs in subjects (e.g. + * `elys-de81bcabef5faf6a - [Nyh5Q07Y]`); only text that still reads like a + * name once those are removed identifies a release. + */ +function isRandomToken(token: string): boolean { + const t = token.replace(/^\[|\]$/g, '') + if (t.length < 6) return false + if (/^[0-9a-f]+$/i.test(t) && /\d/.test(t)) return true + return /\d/.test(t) && /[a-z]/.test(t) && /[A-Z]/.test(t) && !/[._-]/.test(t) +} + +/** + * A stable per-post token: one short run of letters/digits with at least one + * digit. Requiring a digit keeps constant labels like `[PRiVATE]` from + * grouping different posts of the same poster. + */ +export function isPostToken(remainder: string): boolean { + const compact = remainder.replace(/[\s\[\]()_-]+/g, '') + return compact.length >= 6 && compact.length <= 40 && /\d/.test(compact) && /^[A-Za-z0-9]+$/.test(compact) +} + +export function namesRelease(remainder: string): boolean { + const words = remainder + .split(/\s+/) + .filter((w) => w && !isRandomToken(w)) + .join(' ') + // At least two words of letters, joined the way release names are. + return /[A-Za-z]{2,}[._ -]+[A-Za-z0-9]{2,}/.test(words) +} + +function hash(...parts: (string | number | undefined)[]) { + return createHash('sha1').update(parts.map((p) => p ?? '').join('\n')).digest('hex') +} + +/** + * Decides which release (download) a file belongs to. Returns null when the + * file cannot be grouped from its name alone (random extensionless names in + * obfuscated posts; those need par2 matching). See CONCEPT.md (D16). + */ +export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null { + const counter = /\[(\d+)\/(\d+)\]/.exec(input.subject) + const fileTotal = counter ? Number.parseInt(counter[2], 10) : undefined + const stem = fileStem(input.name) + + if (input.weak) { + // Obfuscating posters randomise the poster per article; a real filename + // stem is specific enough on its own. + if (!looksLikeFilename(input.name)) return null + return { key: hash('weak', stem), name: stem, source: 'weak-stem' } + } + + // Text outside the quotes that names the release, e.g. "Some.Release [01/45] - "x.rar"". + const remainder = subjectRemainder(input.subject) + if (namesRelease(remainder)) { + return { + key: hash('subject', input.poster, remainder), + name: remainder, + fileTotal, + source: 'subject', + } + } + // Not a name, but a post id (e.g. `83110629-n [10/18] - "random"`): with the + // [i/M] counter it groups the files of one post whose names are random. + if (fileTotal && isPostToken(remainder)) { + return { key: hash('post', input.poster, remainder, fileTotal), name: remainder, fileTotal, source: 'post' } + } + // The [i/M] total is informational, not part of the key: posters often + // upload the par2 index without a counter, and it belongs to the same post. + return { key: hash('stem', input.poster, stem), name: stem, fileTotal, source: 'stem' } +} + +/** Release of a par2 recovery set; every file listed in it belongs to it. */ +export function par2ReleaseIdentity(recoverySetId: string, fileNames: string[]): ReleaseIdentity { + return { key: hash('par2', recoverySetId), name: par2ReleaseName(fileNames) ?? recoverySetId, source: 'par2' } +} + +const SEPARATOR = /[._ -]/ + +/** + * A release name from the files a par2 set lists. Files that differ only in + * the middle (episodes S03E01…S03E20, parts) give their shared name, with + * the episode reduced to the season: SEAL.Team.S03E01.1080p…-nitrato.mkv to + * SEAL.Team.S03E20.1080p…-nitrato.mkv give SEAL.Team.S03.1080p…-nitrato. + * Otherwise the longest stem, which tends to be the most descriptive (the + * video file). + */ +export function par2ReleaseName(fileNames: string[]): string | undefined { + const stems = [...new Set(fileNames.map((n) => fileStem(n.split(/[\\/]/).pop() ?? n)))].filter(Boolean) + if (stems.length === 0) return undefined + const longest = [...stems].sort((a, b) => b.length - a.length)[0] + if (stems.length < 2) return longest + + let prefix = stems.reduce((p, s) => { + let i = 0 + while (i < p.length && i < s.length && p[i] === s[i]) i++ + return p.slice(0, i) + }) + let suffix = stems.reduce((p, s) => { + let i = 0 + while (i < p.length && i < s.length && p[p.length - 1 - i] === s[s.length - 1 - i]) i++ + return p.slice(p.length - i) + }) + const shortest = Math.min(...stems.map((s) => s.length)) + if (prefix.length + suffix.length > shortest) suffix = suffix.slice(prefix.length + suffix.length - shortest) + + // Season packs: "Show.S03E" → "Show.S03". Otherwise back to a whole token. + const season = /^(.*[._ -]S\d{1,2})E\d*$/i.exec(prefix) + if (season) prefix = season[1] + else if (prefix.length < stems[0].length && !SEPARATOR.test(stems[0][prefix.length] ?? '.')) { + prefix = prefix.replace(/[^._ -]*$/, '') + } + suffix = SEPARATOR.test(suffix[0] ?? '') ? suffix : suffix.replace(/^[^._ -]*/, '') + prefix = prefix.replace(/[._ -]+$/, '') + + const name = suffix ? prefix + suffix : prefix + // Only when the shared part carries most of the name (not just "The."). + return name.length >= 8 && name.length >= 0.6 * shortest ? name : longest +} + +/** + * Release of a RAR volume set, from what its headers list: the shared + * top-level directory if there is one, otherwise the first entry. For + * volumes after the first, the first entry is the file that continues from + * the previous volume, so all volumes of a single-file archive agree. + */ +export function rarReleaseIdentity(entries: { name: string; size: number | null }[]): ReleaseIdentity | null { + if (entries.length === 0) return null + const tops = new Set(entries.map((e) => e.name.split(/[\\/]/)[0])) + const nested = entries.some((e) => /[\\/]/.test(e.name)) + if (nested && tops.size === 1) { + const [top] = tops + return { key: hash('rar-dir', top), name: top, source: 'rar' } + } + const first = entries[0] + const base = first.name.split(/[\\/]/).pop() ?? first.name + return { key: hash('rar-entry', first.name, first.size ?? ''), name: fileStem(base), source: 'rar' } +} diff --git a/lib/split/index.ts b/lib/split/index.ts new file mode 100644 index 0000000..733d478 --- /dev/null +++ b/lib/split/index.ts @@ -0,0 +1,4 @@ +/** + * Separating equal-sized obfuscated volumes by posting order, independent of AdonisJS. + */ +export { splitInstances, splitByRank, splitVerified, verifyVolume, type SplitArticle, type SplitOptions } from './instances.js' diff --git a/lib/split/instances.ts b/lib/split/instances.ts new file mode 100644 index 0000000..db63795 --- /dev/null +++ b/lib/split/instances.ts @@ -0,0 +1,178 @@ +import { combineCrc32 } from '../yenc/decode.js' + +export interface SplitArticle { + number: number + messageId: string + bytes: number + /** Article number in the group; posting order. */ + articleNumber: number | null + /** yEnc part CRC and decoded part length, for verification. */ + pcrc32?: string | null + partSize?: number | null + /** Whole-file CRC32, usually only in the last part. */ + fileCrc32?: string | null +} + +export interface SplitOptions { + /** Total parts per volume; a volume that has them all takes no more articles. */ + totalParts: number + /** + * Maximum distance in article numbers between consecutive articles of one + * volume. Separates different posts that happen to share a size. + */ + window?: number + /** + * How far (in posting positions) an article may be from where its volume + * predicts it; covers out-of-order finishing across connections. + */ + tolerance?: number +} + +interface Volume { + articles: SplitArticle[] + /** Posting rank of each part number present. */ + ranks: Map + lastArticle: number + opened: number +} + +/** ~1 minute of posting in a busy group. */ +const DEFAULT_WINDOW = 20_000 + +/** + * Separates the articles of several equal-sized files that share one + * identity (obfuscated volumes with random names, possibly from different + * posts) into one list per file. See CONCEPT.md (D20). + * + * Posting tools upload a queue (volume 1 parts 1..n, volume 2 parts 1..n, + * ...) over several connections, so within a volume part k+1 lands about one + * position after part k in posting order, give or take the connection count. + * Each article, in posting order, goes to the eligible volume (lacks this + * part, not complete, active within `window` article numbers) whose + * predicted position for it is closest: taken from that volume's article with + * the nearest part number, which keeps the prediction local and unaffected + * by missing articles elsewhere. A deviation above the tolerance starts a new + * volume. The result is recomputed from all articles, so it improves as late + * articles arrive. + */ +export function splitInstances(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] { + const window = options.window ?? DEFAULT_WINDOW + const tolerance = options.tolerance ?? Math.max(16, Math.ceil(options.totalParts * 0.1)) + const seen = new Set() + const ordered = articles + .filter((a) => (seen.has(a.messageId) ? false : (seen.add(a.messageId), true))) + .sort( + (a, b) => + (a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) || + a.messageId.localeCompare(b.messageId) + ) + + const volumes: Volume[] = [] + ordered.forEach((article, rank) => { + const position = article.articleNumber ?? Number.MAX_SAFE_INTEGER + let best: Volume | undefined + let bestScore: [number, number] | undefined + for (const volume of volumes) { + if (volume.ranks.has(article.number) || volume.ranks.size >= options.totalParts) continue + if (position - volume.lastArticle > window) continue + const deviation = Math.abs(rank - predictedRank(volume, article.number)) + if (deviation > tolerance) continue + const score: [number, number] = [deviation, volume.opened] + if (!bestScore || score[0] < bestScore[0] || (score[0] === bestScore[0] && score[1] < bestScore[1])) { + best = volume + bestScore = score + } + } + if (!best) { + best = { articles: [], ranks: new Map(), lastArticle: position, opened: volumes.length } + volumes.push(best) + } + best.articles.push(article) + best.ranks.set(article.number, rank) + best.lastArticle = Math.max(best.lastArticle, position) + }) + + return volumes.map((v) => v.articles.sort((a, b) => a.number - b.number)) +} + +/** Where part `number` should be in posting order, judged from the volume's nearest part. */ +function predictedRank(volume: Volume, number: number): number { + let anchorPart = 0 + let anchorRank = 0 + let nearest = Number.MAX_SAFE_INTEGER + for (const [part, rank] of volume.ranks) { + const distance = Math.abs(part - number) + if (distance < nearest) { + nearest = distance + anchorPart = part + anchorRank = rank + } + } + return anchorRank + (number - anchorPart) +} + +/** + * Simpler split: for every part number, the i-th article in posting order + * belongs to volume i. Right for a single, complete post; used as a second + * proposal for verification. + */ +export function splitByRank(articles: SplitArticle[]): SplitArticle[][] { + const byNumber = new Map() + const seen = new Set() + for (const article of articles) { + if (seen.has(article.messageId)) continue + seen.add(article.messageId) + byNumber.set(article.number, [...(byNumber.get(article.number) ?? []), article]) + } + const volumes: SplitArticle[][] = [] + for (const group of byNumber.values()) { + group.sort( + (a, b) => + (a.articleNumber ?? Number.MAX_SAFE_INTEGER) - (b.articleNumber ?? Number.MAX_SAFE_INTEGER) || + a.messageId.localeCompare(b.messageId) + ) + group.forEach((article, i) => (volumes[i] ??= []).push(article)) + } + return volumes.map((v) => v.sort((a, b) => a.number - b.number)) +} + +/** + * True when a volume has every part and its combined part CRCs equal the + * whole-file CRC32 its last part announces: then it is exactly one file. + */ +export function verifyVolume(volume: SplitArticle[], totalParts: number): boolean { + if (volume.length !== totalParts) return false + const sorted = [...volume].sort((a, b) => a.number - b.number) + if (sorted.some((a, i) => a.number !== i + 1 || !a.pcrc32 || !a.partSize)) return false + const expected = sorted[sorted.length - 1].fileCrc32 + if (!expected) return false + const combined = combineCrc32(sorted.map((a) => ({ crc: a.pcrc32!, length: a.partSize! }))) + return combined === expected.toLowerCase().padStart(8, '0') +} + +/** + * Splits and keeps only volumes that verify (verifyVolume). Proposals from + * both methods are tried; articles of verified volumes are taken out and the + * rest is tried again, until nothing new verifies. Unverified articles are + * left out: guessing wrong would produce a corrupt file. + */ +export function splitVerified(articles: SplitArticle[], options: SplitOptions): SplitArticle[][] { + const verified: SplitArticle[][] = [] + let remaining = articles + for (let round = 0; round < 4 && remaining.length >= options.totalParts; round++) { + const found: SplitArticle[][] = [] + const used = new Set() + for (const propose of [(a: SplitArticle[]) => splitInstances(a, options), splitByRank]) { + for (const volume of propose(remaining.filter((a) => !used.has(a.messageId)))) { + if (verifyVolume(volume, options.totalParts)) { + found.push(volume) + for (const article of volume) used.add(article.messageId) + } + } + } + if (found.length === 0) break + verified.push(...found) + remaining = remaining.filter((a) => !used.has(a.messageId)) + } + return verified +} diff --git a/lib/spotnet/header.ts b/lib/spotnet/header.ts new file mode 100644 index 0000000..f550483 --- /dev/null +++ b/lib/spotnet/header.ts @@ -0,0 +1,162 @@ +import { createHash, createPublicKey, verify } from 'node:crypto' +import { crc32 } from 'node:zlib' + +/** + * Spotnet spot headers (free.pt), after Spotweb's Services_Format_Parsing::parseHeader. + * + * From: Nick <[PUBKEY[.USERSIGNATURE]]@[CAT][KEYID][SUBCATS].[SIZE].[x].[DATE].[..].[SIGNATURE]> + * Subject: Title | Tag + */ +export interface SpotHeader { + messageId: string + poster: string + title: string + tag: string + /** 0 image/video, 1 sound, 2 games, 3 applications. */ + category: number + keyId: number + /** e.g. { a: [9], b: [3], c: [7, 10], d: [17], z: [0] } */ + subcats: Record + size: number + /** Unix seconds. */ + stamp: number + /** Everything after the @. */ + header: string + headerSign: string + selfSignedPubKey: string + userSignature?: string +} + +/** Spotnet's base64 variant: `-s` for `/`, `-p` for `+`, padding left out. */ +export function spotUnprepareBase64(input: string): string { + const padded = input.length % 4 ? input + '='.repeat(4 - (input.length % 4)) : input + return padded.replace(/-s/g, '/').replace(/-p/g, '+') +} + +/** Parses the From/Subject of a spot. Null when it isn't one. */ +export function parseSpotHeader(from: string, subject: string, messageId: string, date?: string): SpotHeader | null { + const lt = from.indexOf('<') + if (lt === -1) return null + const address = from.slice(lt + 1, from.endsWith('>') ? -1 : undefined) + const at = address.lastIndexOf('@') + if (at === -1) return null + const header = address.slice(at + 1) + const [pubKey, userSignature] = address.slice(0, at).split('.') + + const fields = header.split('.') + if (fields.length < 6) return null + const category = Number.parseInt(fields[0][0], 10) - 1 + const keyId = Number.parseInt(fields[0][1], 10) + if (!(category >= 0) || !(keyId >= 0)) return null + + // a9b4c0d5 (FTD) or a09b04c00d05 (newer): a letter, then digits. + const subcats: Record = {} + for (const [, letter, digits] of fields[0].slice(2).toLowerCase().matchAll(/([a-z])(\d+)/g)) { + if ('abcdz'.includes(letter)) (subcats[letter] ??= []).push(Number.parseInt(digits, 10)) + } + + // Newer keys: "Title | Tag". Key 1 (oldest): the tag is the last field. + const parts = subject.split('|') + let title: string + let tag = '' + if (keyId !== 1) { + title = parts[0].trim() + tag = parts.length > 1 ? parts[1].trim() : '' + } else { + tag = parts.length > 1 ? parts[parts.length - 1].trim() : subject.trim() + title = parts.length > 1 ? parts.slice(0, -2).join('|').trim() : '' + } + + const parsedDate = date ? Date.parse(date) : Number.NaN + return { + messageId: messageId.replace(/^<|>$/g, ''), + poster: from.slice(0, lt).trim(), + title, + tag, + category, + keyId, + subcats, + size: Number.parseInt(fields[1], 10) || 0, + stamp: Number.isNaN(parsedDate) ? Number.parseInt(fields[3], 10) || 0 : Math.floor(parsedDate / 1000), + header, + headerSign: fields[fields.length - 1], + selfSignedPubKey: spotUnprepareBase64(pubKey ?? ''), + userSignature: userSignature ? spotUnprepareBase64(userSignature) : undefined, + } +} + +/** Spotnet's own signing keys (Spotweb settings `rsa_keys`), base64 modulus. */ +export const SPOTNET_KEYS: Record = { + 2: 'ys8WSlqonQMWT8ubG0tAA2Q07P36E+CJmb875wSR1XH7IFhEi0CCwlUzNqBFhC+P', + 3: 'uiyChPV23eguLAJNttC/o0nAsxXgdjtvUvidV2JL+hjNzc4Tc/PPo2JdYvsqUsat', + 4: '1k6RNDVD6yBYWR6kHmwzmSud7JkNV4SMigBrs+jFgOK5Ldzwl17mKXJhl+su/GR9', +} + +const base64url = (b64: string) => Buffer.from(b64, 'base64').toString('base64url') + +/** RSA PKCS#1 v1.5 with SHA-1, as Spotweb's checkRsaSignature (exponent AQAB). */ +export function checkRsaSignature(message: Buffer | string, signatureB64: string, modulusB64: string): boolean { + try { + const key = createPublicKey({ key: { kty: 'RSA', n: base64url(modulusB64), e: 'AQAB' }, format: 'jwk' }) + const data = typeof message === 'string' ? toBytes(message) : message + return verify('sha1', data, key, Buffer.from(signatureB64, 'base64')) + } catch { + return false + } +} + +/** Spots are signed over their ISO-8859-1 bytes. */ +function toBytes(text: string): Buffer { + return /^[\u0000-ÿ]*$/.test(text) ? Buffer.from(text, 'latin1') : Buffer.from(text, 'utf8') +} + +/** SPOTSIGN_V2 and self-signed messages carry a hashcash: sha1('') starts with 0000. */ +function hashcash(messageId: string): boolean { + return createHash('sha1').update(`<${messageId}>`).digest('hex').startsWith('0000') +} + +/** + * Verifies a spot header like Spotweb does: key 7 (self-signed) needs a + * hashcash and the poster's key, keys 2–4 Spotnet's keys; spots that also + * carry a user signature must have it match the message id. + */ +export function verifySpotHeader(spot: SpotHeader, keys: Record = SPOTNET_KEYS): boolean { + if (!spot.title || !spot.poster) return false + // Old key 1 spots from before 2011 weren't signed. + if (spot.keyId === 1 && spot.stamp <= 1293870080) return true + if (!spot.headerSign) return false + + const signature = spotUnprepareBase64(spot.headerSign) + const toCheck = spot.title + spot.header.slice(0, spot.header.length - spot.headerSign.length - 1) + spot.poster + let verified: boolean + if (spot.keyId === 7 || (spot.keyId === 2 && spot.selfSignedPubKey.length > 50)) { + // Self-signed (and key-2 "dispose" messages): the hashcash is the gate, as in Spotweb. + verified = hashcash(spot.messageId) + } else { + verified = keys[spot.keyId] !== undefined && checkRsaSignature(toCheck, signature, keys[spot.keyId]) + } + if (verified && spot.userSignature && spot.selfSignedPubKey) { + verified = checkRsaSignature(`<${spot.messageId}>`, spot.userSignature, spot.selfSignedPubKey) + } + return verified +} + +/** Spotweb's spotter id: base64 of the CRC32 (little-endian) of the poster's public key. */ +export function spotterId(pubKeyB64: string): string { + if (!pubKeyB64) return '' + const crc = Buffer.alloc(4) + crc.writeUInt32LE(crc32(Buffer.from(pubKeyB64, 'base64')) >>> 0) + return crc.toString('base64').replace(/[/+=]/g, '') +} + +/** + * Moderation: key-2 spots titled "delete|dispose|remove ". + * From Spotnet's key they are moderator actions; with the poster's own key + * (a spotter id) a personal dispose of their own spot. + */ +export function moderationTarget(spot: Pick): string | null { + if (spot.keyId !== 2) return null + const [command, target] = spot.title.split(' ') + if (!['delete', 'dispose', 'remove'].includes(command?.toLowerCase()) || !target) return null + return target.replace(/^<|>$/g, '') +} diff --git a/lib/spotnet/index.ts b/lib/spotnet/index.ts new file mode 100644 index 0000000..4d8057a --- /dev/null +++ b/lib/spotnet/index.ts @@ -0,0 +1,7 @@ +/** + * Spotnet (free.pt) spots, independent of AdonisJS: header parsing and + * signature checks after Spotweb, the spot XML and its compressed NZB. + * See CONCEPT.md (D29). + */ +export { parseSpotHeader, verifySpotHeader, spotterId, moderationTarget, checkRsaSignature, spotUnprepareBase64, SPOTNET_KEYS, type SpotHeader } from './header.js' +export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, type SpotXml } from './xml.js' diff --git a/lib/spotnet/xml.ts b/lib/spotnet/xml.ts new file mode 100644 index 0000000..467956b --- /dev/null +++ b/lib/spotnet/xml.ts @@ -0,0 +1,84 @@ +import { inflateRawSync } from 'node:zlib' + +export interface SpotXml { + title: string + category: number | null + subcats: string[] + size: number + website: string + /** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */ + nzbSegments: string[] + created: number | null +} + +const ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" } + +function text(value: string | undefined): string { + if (value === undefined) return '' + const cdata = /^\s*\s*$/.exec(value) + if (cdata) return cdata[1] + return value.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => { + if (entity[0] === '#') { + const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10) + return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match + } + return ENTITIES[entity.toLowerCase()] ?? match + }) +} + +const element = (xml: string, name: string) => new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)`, 'i').exec(xml)?.[1] + +/** The spot XML from the X-XML header lines (a tolerant scanner, as spots are often invalid XML). */ +export function parseSpotXml(xml: string): SpotXml | null { + const posting = element(xml, 'Posting') + if (!posting) return null + const categoryXml = element(posting, 'Category') ?? '' + const key = Number.parseInt(element(posting, 'Key') ?? '', 10) + const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10) + const nzb = element(posting, 'NZB') ?? '' + return { + title: text(element(posting, 'Title')).trim(), + // Newer keys number categories from 1. + category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory, + subcats: [...categoryXml.matchAll(/<(?:Sub|SubCat)>([^<]*)<\/(?:Sub|SubCat)>/gi)].map((m) => m[1].trim()), + size: Number.parseInt(element(posting, 'Size') ?? '', 10) || 0, + website: text(element(posting, 'Website')).trim(), + nzbSegments: [...nzb.matchAll(/([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean), + created: Number.parseInt(element(posting, 'Created') ?? '', 10) || null, + } +} + +/** Header lines of a raw HEAD response; X-XML (split over many lines) joined. */ +export function parseSpotArticleHeaders(raw: string): { headers: Record; xml: string } { + const headers: Record = {} + let xml = '' + for (const line of raw.split(/\r?\n/)) { + const colon = line.indexOf(':') + if (colon <= 0) continue + const name = line.slice(0, colon).toLowerCase() + const value = line.slice(colon + 1).replace(/^ /, '') + if (name === 'x-xml') xml += value + else headers[name] ??= value + } + return { headers, xml } +} + +/** + * Spot binaries (the NZB, images): article bodies joined without line + * breaks, Spotnet's escapes undone (=A NUL, =B CR, =C LF, =D =), then raw + * deflate (PHP gzinflate). + */ +export function decodeSpotBinary(bodies: Buffer[], compressed = true): Buffer { + const joined = bodies + .map((body) => + body + .toString('latin1') + .split('\r\n') + // NNTP dot-stuffing. + .map((line) => (line.startsWith('..') ? line.slice(1) : line)) + .join('') + ) + .join('') + const unescaped = Buffer.from(joined.replace(/=([ABCD])/g, (_, c: string) => ({ A: '\0', B: '\r', C: '\n', D: '=' })[c]!), 'latin1') + return compressed ? inflateRawSync(unescaped) : unescaped +} diff --git a/lib/subject/file_key.ts b/lib/subject/file_key.ts new file mode 100644 index 0000000..45841b2 --- /dev/null +++ b/lib/subject/file_key.ts @@ -0,0 +1,30 @@ +import { createHash } from 'node:crypto' + +/** + * Identity of one uploaded file: every segment of it has the same poster, the + * same subject apart from the segment counter, and the same segment count. + * The group is deliberately left out so crossposts resolve to one file. + * See CONCEPT.md (D12). + */ +export function fileKey(poster: string, base: string, totalSegments: number): string { + return createHash('sha1').update(`${poster}\n${base}\n${totalSegments}`).digest('hex') +} + +/** True when a yEnc name looks like a real filename (it has an extension), not a random token. */ +export function looksLikeFilename(name: string): boolean { + return /\.[A-Za-z0-9]{1,5}$/.test(name) && !/^[0-9a-f]{32,}$/i.test(name) +} + +/** + * Identity of a file known only from its yEnc header (obfuscated posts). + * The poster is left out: obfuscating posters randomise it per article. + * - A real filename is stable across the articles of a file, and needed to + * tell apart the equal-sized volumes of one archive. + * - A random, extensionless name can change per article too; the exact + * byte size and part count are what stays the same. + * Collisions are caught by the conflict check in IngestService. + */ +export function weakFileKey(name: string, size: number, totalParts: number): string { + const identity = looksLikeFilename(name) ? `name\n${name}` : 'shape' + return createHash('sha1').update(`weak\n${identity}\n${size}\n${totalParts}`).digest('hex') +} diff --git a/lib/subject/index.ts b/lib/subject/index.ts new file mode 100644 index 0000000..38f5aa3 --- /dev/null +++ b/lib/subject/index.ts @@ -0,0 +1,5 @@ +/** + * Conventions of binary posts (subject lines, file identity), independent of AdonisJS. + */ +export { parseSubject, type ParsedSubject } from './parse.js' +export { fileKey, weakFileKey, looksLikeFilename } from './file_key.js' diff --git a/lib/subject/parse.ts b/lib/subject/parse.ts new file mode 100644 index 0000000..8304476 --- /dev/null +++ b/lib/subject/parse.ts @@ -0,0 +1,59 @@ +export interface ParsedSubject { + /** Filename from the quoted part of the subject. */ + name: string + /** Segment number `k` from `(k/N)`. */ + segment: number + /** Segment count `N` from `(k/N)`. */ + totalSegments: number + /** File counter `[i/M]`, when present. */ + fileIndex?: number + fileTotal?: number + /** + * The subject with the segment counter removed. Identical for every segment + * of one uploaded file, so it is part of the file identity (see fileKey). + */ + base: string +} + +const SEGMENT_COUNTER = /\((\d+)\/(\d+)\)/g +const FILE_COUNTER = /\[(\d+)\/(\d+)\]/ + +/** + * Parses a binary post subject such as + * `[07/45] - "Some.Release.part06.rar" yEnc (12/137) 51315352`. + * + * Returns null for subjects without a quoted filename and segment counter + * (obfuscated posts), which need their yEnc header read from the body instead. + */ +export function parseSubject(subject: string): ParsedSubject | null { + const nameEnd = subject.lastIndexOf('"') + const nameStart = nameEnd > 0 ? subject.lastIndexOf('"', nameEnd - 1) : -1 + if (nameStart === -1) return null + const name = subject.slice(nameStart + 1, nameEnd) + if (!name) return null + + // The segment counter is the last "(k/N)" after the filename; a file size may follow it. + let counter: RegExpExecArray | undefined + for (const match of subject.matchAll(SEGMENT_COUNTER)) { + if (match.index > nameEnd) counter = match + } + if (!counter) return null + + const segment = Number.parseInt(counter[1], 10) + const totalSegments = Number.parseInt(counter[2], 10) + if (totalSegments < 1 || segment < 1 || segment > totalSegments) return null + + const base = (subject.slice(0, counter.index) + subject.slice(counter.index + counter[0].length)) + .replace(/\s+/g, ' ') + .trim() + + const fileCounter = FILE_COUNTER.exec(subject.slice(0, nameStart)) + return { + name, + segment, + totalSegments, + fileIndex: fileCounter ? Number.parseInt(fileCounter[1], 10) : undefined, + fileTotal: fileCounter ? Number.parseInt(fileCounter[2], 10) : undefined, + base, + } +} diff --git a/lib/yenc/decode.ts b/lib/yenc/decode.ts new file mode 100644 index 0000000..60e5dc0 --- /dev/null +++ b/lib/yenc/decode.ts @@ -0,0 +1,76 @@ +import yencode from 'yencode' +import { YencCrcError, YencError } from './errors.js' +import { parseYencMeta, type YencMeta } from './meta.js' + +export interface DecodedSegment { + meta: YencMeta + /** Decoded bytes of this segment. */ + data: Buffer + /** 0-based offset of `data` within the complete file. */ + offset: number +} + +export interface DecodeOptions { + /** + * Undo NNTP dot-stuffing (`..` at line start). nntp-js hands back raw + * bodies, so this defaults to true. + */ + stripDots?: boolean + /** Verify the part (or whole-file) CRC32 from the `=yend` line. Default true. */ + verifyCrc?: boolean + /** + * Accept a decoded length that differs from the declared size (the CRC is + * still checked). Some posting tools declare a size that leaves out a + * header, e.g. the 20-byte header of `nzbe` containers. + */ + lenientSize?: boolean +} + +export function crc32Hex(data: Buffer): string { + return yencode.crc32(data).toString('hex') +} + +/** + * Whole-file CRC32 from the CRCs and decoded lengths of its parts, in order, + * without the data itself. + */ +export function combineCrc32(parts: { crc: string; length: number }[]): string { + let combined: Buffer | undefined + for (const part of parts) { + const crc = Buffer.from(part.crc.padStart(8, '0'), 'hex') + combined = combined ? yencode.crc32_combine(combined, crc, part.length) : crc + } + return (combined ?? Buffer.alloc(4)).toString('hex') +} + +/** + * Decodes one yEnc article body and checks its size and CRC32 against the + * `=yend` trailer. + */ +export function decodeSegment(body: Buffer, options: DecodeOptions = {}): DecodedSegment { + const { stripDots = true, verifyCrc = true, lenientSize = false } = options + const meta = parseYencMeta(body) + const data = yencode.decode(body.subarray(meta.dataStart, meta.dataEnd), stripDots) + + if (lenientSize) { + // Only the CRC below decides. + } else if (data.length !== meta.end.size) { + throw new YencError(`Decoded size ${data.length} does not match =yend size ${meta.end.size}`) + } + if (!lenientSize && meta.part && meta.part.end - meta.part.begin + 1 !== data.length) { + throw new YencError( + `Decoded size ${data.length} does not match =ypart range ${meta.part.begin}-${meta.part.end}` + ) + } + + if (verifyCrc) { + // Multi-part posts carry pcrc32 per part; single-part posts only crc32. + const expected = meta.end.pcrc32 ?? (meta.part ? undefined : meta.end.crc32) + if (expected) { + const actual = crc32Hex(data) + if (actual !== expected) throw new YencCrcError(expected, actual) + } + } + + return { meta, data, offset: meta.part ? meta.part.begin - 1 : 0 } +} diff --git a/lib/yenc/errors.ts b/lib/yenc/errors.ts new file mode 100644 index 0000000..bc95c63 --- /dev/null +++ b/lib/yenc/errors.ts @@ -0,0 +1,16 @@ +export class YencError extends Error { + constructor(message: string) { + super(message) + this.name = 'YencError' + } +} + +export class YencCrcError extends YencError { + constructor( + public readonly expected: string, + public readonly actual: string + ) { + super(`CRC32 mismatch: expected ${expected}, got ${actual}`) + this.name = 'YencCrcError' + } +} diff --git a/lib/yenc/index.ts b/lib/yenc/index.ts new file mode 100644 index 0000000..1d13cd9 --- /dev/null +++ b/lib/yenc/index.ts @@ -0,0 +1,7 @@ +/** + * yEnc decoding, independent of AdonisJS. See CONCEPT.md (D4, D10). + */ +export { parseYencMeta, inferTotalParts, type YencMeta, type YencBegin, type YencPart, type YencEnd } from './meta.js' +export { decodeSegment, crc32Hex, combineCrc32, type DecodedSegment, type DecodeOptions } from './decode.js' +export { YencFile } from './yenc_file.js' +export { YencError, YencCrcError } from './errors.js' diff --git a/lib/yenc/meta.ts b/lib/yenc/meta.ts new file mode 100644 index 0000000..3046b6c --- /dev/null +++ b/lib/yenc/meta.ts @@ -0,0 +1,140 @@ +import { YencError } from './errors.js' + +/** Fields of the `=ybegin` line. `part`/`total` are absent in single-part posts. */ +export interface YencBegin { + line: number + size: number + name: string + part?: number + total?: number +} + +/** Fields of the `=ypart` line. Offsets are 1-based and `end` is inclusive. */ +export interface YencPart { + begin: number + end: number +} + +/** Fields of the `=yend` line. CRCs are lowercase 8-digit hex. */ +export interface YencEnd { + size: number + part?: number + pcrc32?: string + crc32?: string +} + +export interface YencMeta { + begin: YencBegin + part?: YencPart + end: YencEnd + /** Byte range of the encoded data inside the article body. */ + dataStart: number + dataEnd: number +} + +const CRLF = Buffer.from('\r\n') +const BEGIN = Buffer.from('=ybegin ') +const PART = Buffer.from('=ypart ') +const END = Buffer.from('\r\n=yend ') + +/** + * Parses `key=value` pairs from a yEnc header line. `name` is always the last + * field and runs to the end of the line, because filenames may contain spaces + * and `=`. + */ +function parseFields(line: string): Record { + const fields: Record = {} + const nameIndex = line.search(/(^|\s)name=/) + let rest = line + if (nameIndex !== -1) { + const nameStart = line.indexOf('name=', nameIndex) + 'name='.length + fields.name = line.slice(nameStart).trim() + rest = line.slice(0, nameIndex) + } + for (const match of rest.matchAll(/(\w+)=(\S+)/g)) { + fields[match[1]] = match[2] + } + return fields +} + +function toInt(fields: Record, key: string, required: true): number +function toInt(fields: Record, key: string, required?: false): number | undefined +function toInt(fields: Record, key: string, required = false) { + const raw = fields[key] + if (raw === undefined) { + if (required) throw new YencError(`Missing "${key}" in yEnc header`) + return undefined + } + const value = Number.parseInt(raw, 10) + if (!Number.isFinite(value)) throw new YencError(`Invalid "${key}" in yEnc header: ${raw}`) + return value +} + +function toCrc(fields: Record, key: string) { + const raw = fields[key] + if (raw === undefined) return undefined + if (!/^[0-9a-fA-F]{1,8}$/.test(raw)) throw new YencError(`Invalid "${key}" in yEnc trailer: ${raw}`) + return raw.toLowerCase().padStart(8, '0') +} + +/** + * Number of parts in the file. Many posters leave `total=` out of `=ybegin`; + * then it follows from the part size, since all parts except the last one + * are the same size. + */ +export function inferTotalParts(meta: Pick): number { + const { begin, part } = meta + if (begin.total) return begin.total + if (!part || begin.part === undefined) return 1 + if (part.end >= begin.size) return begin.part + return Math.ceil(begin.size / (part.end - part.begin + 1)) +} + +function lineAt(buffer: Buffer, start: number) { + const eol = buffer.indexOf(CRLF, start) + const end = eol === -1 ? buffer.length : eol + return { text: buffer.subarray(start, end).toString('latin1'), next: end + CRLF.length } +} + +/** + * Locates and parses the yEnc header/trailer lines of one article body, + * without decoding the data itself. + */ +export function parseYencMeta(body: Buffer): YencMeta { + const beginIndex = body.indexOf(BEGIN) + if (beginIndex === -1) throw new YencError('No =ybegin line found') + + const beginLine = lineAt(body, beginIndex) + const beginFields = parseFields(beginLine.text) + const begin: YencBegin = { + line: toInt(beginFields, 'line', true), + size: toInt(beginFields, 'size', true), + name: beginFields.name ?? '', + part: toInt(beginFields, 'part'), + total: toInt(beginFields, 'total'), + } + if (!begin.name) throw new YencError('Missing "name" in =ybegin line') + + let dataStart = beginLine.next + let part: YencPart | undefined + if (body.subarray(dataStart, dataStart + PART.length).equals(PART)) { + const partLine = lineAt(body, dataStart) + const partFields = parseFields(partLine.text) + part = { begin: toInt(partFields, 'begin', true), end: toInt(partFields, 'end', true) } + dataStart = partLine.next + } + + const endIndex = body.lastIndexOf(END) + if (endIndex === -1 || endIndex < dataStart - CRLF.length) { + throw new YencError('No =yend line found') + } + const endFields = parseFields(lineAt(body, endIndex + CRLF.length).text) + const end: YencEnd = { + size: toInt(endFields, 'size', true), + part: toInt(endFields, 'part'), + pcrc32: toCrc(endFields, 'pcrc32'), + crc32: toCrc(endFields, 'crc32'), + } + + return { begin, part, end, dataStart, dataEnd: Math.max(dataStart, endIndex) } +} diff --git a/lib/yenc/yenc_file.ts b/lib/yenc/yenc_file.ts new file mode 100644 index 0000000..0016565 --- /dev/null +++ b/lib/yenc/yenc_file.ts @@ -0,0 +1,86 @@ +import { crc32Hex, type DecodedSegment } from './decode.js' +import { YencCrcError, YencError } from './errors.js' +import { inferTotalParts } from './meta.js' + +/** + * Assembles the decoded segments of one file into a single buffer. Segments + * may be added in any order. + */ +export class YencFile { + readonly name: string + readonly size: number + readonly totalParts: number + + private readonly buffer: Buffer + private readonly received = new Set() + private bytesWritten = 0 + private fileCrc?: string + + constructor(name: string, size: number, totalParts = 1) { + this.name = name + this.size = size + this.totalParts = totalParts + this.buffer = Buffer.alloc(size) + } + + /** Creates a file sized from the metadata of any of its segments. */ + static fromSegment(segment: DecodedSegment): YencFile { + const { begin } = segment.meta + // Many posters leave total= out; it follows from the part size. + return new YencFile(begin.name, begin.size, inferTotalParts(segment.meta)) + } + + add(segment: DecodedSegment): void { + const { meta, data, offset } = segment + // Only the size is compared: obfuscating posters can give every segment + // of one file a different random name. + if (meta.begin.size !== this.size) { + throw new YencError( + `Segment of "${meta.begin.name}" (${meta.begin.size} bytes) does not belong to "${this.name}" (${this.size} bytes)` + ) + } + if (offset + data.length > this.size) { + throw new YencError(`Segment at offset ${offset} overruns file size ${this.size}`) + } + + const partNumber = meta.begin.part ?? 1 + if (this.received.has(partNumber)) return + + data.copy(this.buffer, offset) + this.received.add(partNumber) + this.bytesWritten += data.length + if (meta.end.crc32) this.fileCrc = meta.end.crc32 + } + + get partsReceived(): number { + return this.received.size + } + + get isComplete(): boolean { + return this.received.size === this.totalParts && this.bytesWritten === this.size + } + + /** Part numbers (1-based) that have not been added yet. */ + missingParts(): number[] { + const missing: number[] = [] + for (let part = 1; part <= this.totalParts; part++) { + if (!this.received.has(part)) missing.push(part) + } + return missing + } + + /** + * Returns the assembled file. Throws if parts are missing or, when the poster + * included a whole-file crc32, if it does not match. + */ + getBuffer(): Buffer { + if (!this.isComplete) { + throw new YencError(`File "${this.name}" is incomplete, missing parts: ${this.missingParts().join(', ')}`) + } + if (this.fileCrc) { + const actual = crc32Hex(this.buffer) + if (actual !== this.fileCrc) throw new YencCrcError(this.fileCrc, actual) + } + return this.buffer + } +} diff --git a/package-lock.json b/package-lock.json index d788ff7..35e9059 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,20 +15,18 @@ "@adonisjs/lucid": "^22.4.2", "@adonisjs/session": "^8.1.0", "@adonisjs/shield": "^9.0.0", + "@adonisjs/static": "^2.0.1", "@japa/api-client": "^3.2.1", "@tuyau/core": "^1.2.2", "@vinejs/vine": "^4.4.0", "better-sqlite3": "^12.10.0", "bullmq": "^5.78.0", - "crc": "^4.3.2", "ioredis": "5.11.0", "log4js": "^6.9.1", "luxon": "^3.7.2", "nntp-js": "^1.0.4", - "node-unrar-js": "^2.0.2", "pg": "^8.21.0", "reflect-metadata": "^0.2.2", - "simple-yenc": "^1.0.4", "xmlbuilder2": "^4.0.3", "yencode": "^1.2.4" }, @@ -707,6 +705,27 @@ } } }, + "node_modules/@adonisjs/static": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@adonisjs/static/-/static-2.0.1.tgz", + "integrity": "sha512-pXSyQgha1yfyQDXPp0ywv7hJqk4JgCCkzWLkku2PW3Y6i/9GUTyzzNTCpz29loZzSF0b/EpiHXQE/sqBhW+p7w==", + "license": "MIT", + "dependencies": { + "serve-static": "^2.2.1" + }, + "engines": { + "node": ">=24.0.0" + }, + "peerDependencies": { + "@adonisjs/assembler": "^8.0.0-next.7 || ^8.0.0", + "@adonisjs/core": "^7.0.0-next.0 || ^7.0.0" + }, + "peerDependenciesMeta": { + "@adonisjs/assembler": { + "optional": true + } + } + }, "node_modules/@adonisjs/tsconfig": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@adonisjs/tsconfig/-/tsconfig-2.0.0.tgz", @@ -2749,6 +2768,30 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bl/node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, "node_modules/brace-expansion": { "version": "5.0.6", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", @@ -2809,30 +2852,6 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, - "node_modules/buffer": { - "version": "5.7.1", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", - "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.1.13" - } - }, "node_modules/builtin-modules": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-5.2.0.tgz", @@ -3336,23 +3355,6 @@ "url": "https://opencollective.com/core-js" } }, - "node_modules/crc": { - "version": "4.3.2", - "resolved": "https://registry.npmjs.org/crc/-/crc-4.3.2.tgz", - "integrity": "sha512-uGDHf4KLLh2zsHa8D8hIQ1H/HtFQhyHrc0uhHBcoKGol/Xnb+MPYfUMw7cvON6ze/GUESTudKayDcJC5HnJv1A==", - "license": "MIT", - "engines": { - "node": ">=12" - }, - "peerDependencies": { - "buffer": ">=6.0.3" - }, - "peerDependenciesMeta": { - "buffer": { - "optional": true - } - } - }, "node_modules/cron-parser": { "version": "4.9.0", "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-4.9.0.tgz", @@ -3776,6 +3778,12 @@ "node": ">=6" } }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, "node_modules/escape-string-regexp": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", @@ -5789,15 +5797,6 @@ "node": ">=18" } }, - "node_modules/node-unrar-js": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/node-unrar-js/-/node-unrar-js-2.0.2.tgz", - "integrity": "sha512-hLNmoJzqaKJnod8yiTVGe9hnlNRHotUi0CreSv/8HtfRi/3JnRC8DvsmKfeGGguRjTEulhZK6zXX5PXoVuDZ2w==", - "license": "MIT", - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/normalize-package-data": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-8.0.0.tgz", @@ -6056,6 +6055,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/path-browserify": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", @@ -6577,6 +6585,19 @@ "node": ">= 0.8" } }, + "node_modules/range-parser": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.3.0.tgz", + "integrity": "sha512-hek2mFQpPuI4E1BBKrSto+BU3e3x4xuarsbiwr3+lf7p44juvFMV0XFWQAP3xUyqXA4RrXLIoaSUGbSt056ZMw==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/raw-body": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-3.0.2.tgz", @@ -6955,6 +6976,60 @@ "node": ">=10" } }, + "node_modules/send": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/send/-/send-1.2.1.tgz", + "integrity": "sha512-1gnZf7DFcoIcajTjTwjwuDjzuz4PPcY2StKPlsGAQ1+YH20IRVrBaXSWmdjowTJ6u8Rc01PoYOGHXfP1mYcZNQ==", + "license": "MIT", + "dependencies": { + "debug": "^4.4.3", + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "etag": "^1.8.1", + "fresh": "^2.0.0", + "http-errors": "^2.0.1", + "mime-types": "^3.0.2", + "ms": "^2.1.3", + "on-finished": "^2.4.1", + "range-parser": "^1.2.1", + "statuses": "^2.0.2" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/send/node_modules/fresh": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-2.0.0.tgz", + "integrity": "sha512-Rx/WycZ60HOaqLKAi6cHRKKI7zxWbJ31MhntmtwMoaTeF7XFH9hhBp8vITaMidfljRQ6eYWCKkaTK+ykVJHP2A==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/serve-static": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-2.2.1.tgz", + "integrity": "sha512-xRXBn0pPqQTVQiC8wyQrKs2MOlX24zQ0POGaj0kultvoOCstBQM5yvOhAVSUwOMjQtTvsPWoNCHfPGwaaQJhTw==", + "license": "MIT", + "dependencies": { + "encodeurl": "^2.0.0", + "escape-html": "^1.0.3", + "parseurl": "^1.3.3", + "send": "^1.2.0" + }, + "engines": { + "node": ">= 18" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, "node_modules/set-cookie-parser": { "version": "2.7.2", "resolved": "https://registry.npmjs.org/set-cookie-parser/-/set-cookie-parser-2.7.2.tgz", @@ -7119,16 +7194,6 @@ "simple-concat": "^1.0.0" } }, - "node_modules/simple-yenc": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/simple-yenc/-/simple-yenc-1.0.4.tgz", - "integrity": "sha512-5gvxpSd79e9a3V4QDYUqnqxeD4HGlhCakVpb6gMnDD7lexJggSBJRBO5h52y/iJrdXRilX9UCuDaIJhSWm5OWw==", - "license": "MIT", - "funding": { - "type": "individual", - "url": "https://github.com/sponsors/eshaz" - } - }, "node_modules/slash": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/slash/-/slash-5.1.0.tgz", diff --git a/package.json b/package.json index cf21df8..a874a4a 100644 --- a/package.json +++ b/package.json @@ -35,7 +35,8 @@ "#database/*": "./database/*.js", "#tests/*": "./tests/*.js", "#start/*": "./start/*.js", - "#config/*": "./config/*.js" + "#config/*": "./config/*.js", + "#lib/*": "./lib/*.js" }, "devDependencies": { "@adonisjs/assembler": "^8.4.0", @@ -62,20 +63,18 @@ "@adonisjs/lucid": "^22.4.2", "@adonisjs/session": "^8.1.0", "@adonisjs/shield": "^9.0.0", + "@adonisjs/static": "^2.0.1", "@japa/api-client": "^3.2.1", "@tuyau/core": "^1.2.2", "@vinejs/vine": "^4.4.0", "better-sqlite3": "^12.10.0", "bullmq": "^5.78.0", - "crc": "^4.3.2", "ioredis": "5.11.0", "log4js": "^6.9.1", "luxon": "^3.7.2", "nntp-js": "^1.0.4", - "node-unrar-js": "^2.0.2", "pg": "^8.21.0", "reflect-metadata": "^0.2.2", - "simple-yenc": "^1.0.4", "xmlbuilder2": "^4.0.3", "yencode": "^1.2.4" }, diff --git a/public/app.css b/public/app.css new file mode 100644 index 0000000..2a23f25 --- /dev/null +++ b/public/app.css @@ -0,0 +1,708 @@ +:root { + color-scheme: light dark; + --bg: #f6f7f9; + --surface: #ffffff; + --surface-2: #f0f2f5; + --border: #e1e4ea; + --text: #1b1f27; + --muted: #5f6878; + --faint: #8a93a3; + --accent: #2f6fed; + --accent-soft: #e7eefe; + --good: #1f8a4c; + --good-soft: #e3f4ea; + --warn: #9a6400; + --warn-soft: #fbf0d9; + --bad: #b3261e; + --bad-soft: #fbe6e4; + --shadow: 0 10px 30px rgba(20, 28, 45, 0.12); + --radius: 10px; + --mono: ui-monospace, 'SF Mono', 'Cascadia Code', 'JetBrains Mono', Menlo, Consolas, monospace; + --sans: system-ui, -apple-system, 'Segoe UI', Roboto, 'Helvetica Neue', sans-serif; +} + +@media (prefers-color-scheme: dark) { + :root { + --bg: #0f1216; + --surface: #161a20; + --surface-2: #1d222a; + --border: #2a303a; + --text: #e6e9ef; + --muted: #a0a8b6; + --faint: #737c8c; + --accent: #6e9bff; + --accent-soft: #1c2740; + --good: #5fcf8e; + --good-soft: #15291e; + --warn: #e8b457; + --warn-soft: #2d2413; + --bad: #f28b82; + --bad-soft: #321a19; + --shadow: 0 10px 30px rgba(0, 0, 0, 0.45); + } +} + +* { + box-sizing: border-box; +} + +/* Elements with their own display value must still honour the hidden attribute. */ +[hidden] { + display: none !important; +} + +html, +body { + margin: 0; + background: var(--bg); + color: var(--text); + font: 15px/1.45 var(--sans); +} + +button, +input, +select { + font: inherit; + color: inherit; +} + +a { + color: var(--accent); +} + +svg { + width: 18px; + height: 18px; + fill: none; + stroke: currentColor; + stroke-width: 2; + stroke-linecap: round; + stroke-linejoin: round; + flex: none; +} + +.visually-hidden { + position: absolute; + width: 1px; + height: 1px; + overflow: hidden; + clip: rect(0 0 0 0); + white-space: nowrap; +} + +/* --- Top bar ----------------------------------------------------------- */ + +.topbar { + display: flex; + align-items: center; + justify-content: space-between; + gap: 16px; + flex-wrap: wrap; + padding: 14px 24px; + background: var(--surface); + border-bottom: 1px solid var(--border); +} + +.brand { + display: flex; + align-items: center; + gap: 10px; + font-weight: 650; + color: var(--text); + text-decoration: none; + letter-spacing: -0.01em; +} + +.brand-mark { + width: 22px; + height: 22px; + border-radius: 6px; + background: + linear-gradient(var(--surface), var(--surface)) 5px 6px / 12px 2px no-repeat, + linear-gradient(var(--surface), var(--surface)) 5px 10px / 12px 2px no-repeat, + linear-gradient(var(--surface), var(--surface)) 5px 14px / 8px 2px no-repeat, + var(--accent); +} + +.stats { + display: flex; + gap: 22px; + margin: 0; + flex-wrap: wrap; +} + +.stats div { + display: flex; + flex-direction: column; +} + +.stats dt { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--faint); +} + +.stats dd { + margin: 0; + font-weight: 600; + font-variant-numeric: tabular-nums; +} + +/* --- Search ------------------------------------------------------------ */ + +.layout { + max-width: 1180px; + margin: 0 auto; + padding: 22px 24px 60px; +} + +.search-row { + display: flex; + gap: 10px; +} + +.search-box { + flex: 1; + display: flex; + align-items: center; + gap: 10px; + padding: 0 14px; + height: 48px; + background: var(--surface); + border: 1px solid var(--border); + border-radius: var(--radius); + color: var(--muted); + transition: border-color 0.15s, box-shadow 0.15s; +} + +.search-box:focus-within { + border-color: var(--accent); + box-shadow: 0 0 0 3px var(--accent-soft); +} + +.search-box input { + flex: 1; + min-width: 0; + border: 0; + outline: 0; + background: transparent; + font-size: 16px; + color: var(--text); +} + +kbd { + font: 12px var(--mono); + padding: 1px 6px; + border: 1px solid var(--border); + border-radius: 5px; + color: var(--faint); +} + +.select select { + height: 48px; + padding: 0 34px 0 12px; + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--surface) + url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24' fill='none' stroke='%238a93a3' stroke-width='2'%3E%3Cpath d='m6 9 6 6 6-6'/%3E%3C/svg%3E") + no-repeat right 10px center / 16px; + appearance: none; + cursor: pointer; +} + +.select-small select { + height: 32px; + border-radius: 999px; + font-size: 13px; +} + +.filters { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; + margin-top: 12px; +} + +.chip { + display: inline-flex; + align-items: center; + gap: 6px; + height: 32px; + padding: 0 13px; + border: 1px solid var(--border); + border-radius: 999px; + background: var(--surface); + font-size: 13px; + cursor: pointer; +} + +.chip[aria-pressed='true'], +.chip-active { + border-color: var(--accent); + background: var(--accent-soft); + color: var(--accent); +} + +.chip-active button { + border: 0; + background: none; + cursor: pointer; + font-size: 16px; + line-height: 1; + color: inherit; + padding: 0 0 0 2px; +} + +/* --- Results ----------------------------------------------------------- */ + +.summary { + margin: 20px 2px 10px; + color: var(--muted); + font-size: 13px; +} + +.release-list { + list-style: none; + margin: 0; + padding: 0; + background: var(--surface); + border: 1px solid var(--border); + border-radius: var(--radius); + overflow: hidden; +} + +.release-list:empty { + display: none; +} + +.release { + display: flex; + align-items: center; + gap: 14px; + border-top: 1px solid var(--border); +} + +.release:first-child { + border-top: 0; +} + +.release:hover, +.release.selected { + background: var(--surface-2); +} + +.release-main { + flex: 1; + min-width: 0; + display: flex; + flex-direction: column; + gap: 3px; + padding: 12px 0 12px 16px; + border: 0; + background: none; + text-align: left; + cursor: pointer; +} + +.release-main:focus-visible { + outline: 2px solid var(--accent); + outline-offset: -2px; +} + +.release-name { + font-weight: 560; + overflow-wrap: anywhere; +} + +.release-name.obfuscated { + font-family: var(--mono); + font-size: 13px; + font-weight: 450; + color: var(--muted); +} + +.release-meta { + font-size: 12.5px; + color: var(--faint); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.release-side { + display: flex; + align-items: center; + gap: 12px; + padding-right: 10px; + flex: none; +} + +.release-size { + width: 76px; + text-align: right; + font-variant-numeric: tabular-nums; + font-size: 13.5px; + color: var(--muted); +} + +.badges { + display: flex; + gap: 6px; + justify-content: flex-end; +} + +.badge { + display: inline-flex; + align-items: center; + gap: 4px; + height: 22px; + padding: 0 8px; + border-radius: 6px; + font-size: 11.5px; + font-weight: 560; + white-space: nowrap; + background: var(--surface-2); + color: var(--muted); +} + +.badge-good { + background: var(--good-soft); + color: var(--good); +} + +.badge-warn { + background: var(--warn-soft); + color: var(--warn); +} + +.badge-bad { + background: var(--bad-soft); + color: var(--bad); +} + +.badge-accent { + background: var(--accent-soft); + color: var(--accent); +} + +.icon-button { + display: inline-grid; + place-items: center; + width: 34px; + height: 34px; + border: 1px solid transparent; + border-radius: 8px; + background: none; + color: var(--muted); + cursor: pointer; +} + +.icon-button:hover { + border-color: var(--border); + background: var(--surface); + color: var(--text); +} + +.more { + display: flex; + justify-content: center; + margin-top: 16px; +} + +.button { + display: inline-flex; + align-items: center; + gap: 8px; + height: 38px; + padding: 0 16px; + border: 1px solid var(--border); + border-radius: 8px; + background: var(--surface); + font-weight: 560; + font-size: 14px; + text-decoration: none; + color: var(--text); + cursor: pointer; +} + +.button:hover { + border-color: var(--faint); +} + +.button-primary { + background: var(--accent); + border-color: var(--accent); + color: #fff; +} + +.button-primary:hover { + filter: brightness(1.07); + border-color: var(--accent); +} + +.empty { + text-align: center; + color: var(--muted); + padding: 40px 0; +} + +/* --- Detail drawer ----------------------------------------------------- */ + +.scrim { + position: fixed; + inset: 0; + background: rgba(10, 14, 20, 0.35); + z-index: 10; +} + +.drawer { + position: fixed; + top: 0; + right: 0; + bottom: 0; + width: min(640px, 100%); + display: flex; + flex-direction: column; + background: var(--surface); + border-left: 1px solid var(--border); + box-shadow: var(--shadow); + z-index: 11; + animation: slide-in 0.18s ease-out; +} + +@keyframes slide-in { + from { + transform: translateX(24px); + opacity: 0; + } +} + +.drawer-head { + display: flex; + align-items: flex-start; + gap: 12px; + padding: 20px 20px 14px 24px; + border-bottom: 1px solid var(--border); +} + +.drawer-title { + flex: 1; + min-width: 0; +} + +.eyebrow { + margin: 0 0 4px; + font-size: 11.5px; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--faint); +} + +.drawer h2 { + margin: 0; + font-size: 18px; + line-height: 1.3; + overflow-wrap: anywhere; +} + +.drawer-body { + flex: 1; + overflow-y: auto; + padding: 18px 24px 40px; +} + +.actions { + display: flex; + gap: 8px; + flex-wrap: wrap; + margin-bottom: 18px; +} + +.facts { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(150px, 1fr)); + gap: 12px 18px; + margin: 0 0 22px; +} + +.facts dt { + font-size: 11.5px; + color: var(--faint); +} + +.facts dd { + margin: 2px 0 0; + font-weight: 560; + overflow-wrap: anywhere; +} + +.link-button { + border: 0; + padding: 0; + background: none; + color: var(--accent); + cursor: pointer; + text-align: left; + font-weight: 560; +} + +.notice { + display: flex; + gap: 10px; + align-items: flex-start; + padding: 11px 14px; + margin-bottom: 18px; + border-radius: 8px; + font-size: 13.5px; +} + +.notice-bad { + background: var(--bad-soft); + color: var(--bad); +} + +.notice-warn { + background: var(--warn-soft); + color: var(--warn); +} + +.progress { + height: 6px; + margin-top: 8px; + border-radius: 99px; + background: var(--border); + overflow: hidden; +} + +.progress span { + display: block; + height: 100%; + background: currentColor; +} + +.section-title { + display: flex; + align-items: baseline; + justify-content: space-between; + margin: 24px 0 8px; + font-size: 13px; + font-weight: 650; + text-transform: uppercase; + letter-spacing: 0.05em; + color: var(--muted); +} + +.section-title small { + font-weight: 500; + text-transform: none; + letter-spacing: 0; + color: var(--faint); +} + +.file-table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} + +.file-table td { + padding: 7px 8px 7px 0; + border-top: 1px solid var(--border); + vertical-align: top; +} + +.file-table tr:first-child td { + border-top: 0; +} + +.file-table .name { + font-family: var(--mono); + font-size: 12.5px; + overflow-wrap: anywhere; +} + +.file-table .posted { + display: block; + margin-top: 2px; + font-size: 11.5px; + color: var(--faint); +} + +.file-table .num { + text-align: right; + white-space: nowrap; + color: var(--muted); + font-variant-numeric: tabular-nums; +} + +.file-table .state { + width: 22px; + text-align: center; +} + +.ok { + color: var(--good); +} + +.missing { + color: var(--faint); +} + +.skeleton { + height: 14px; + margin: 10px 0; + border-radius: 4px; + background: linear-gradient(90deg, var(--surface-2), var(--border), var(--surface-2)); + background-size: 200% 100%; + animation: shimmer 1.2s infinite linear; +} + +@keyframes shimmer { + to { + background-position: -200% 0; + } +} + +/* --- Small screens ----------------------------------------------------- */ + +@media (max-width: 720px) { + .topbar, + .layout { + padding-left: 14px; + padding-right: 14px; + } + + .stats { + gap: 14px; + } + + .search-row { + flex-direction: column; + } + + .release { + flex-wrap: wrap; + gap: 0; + } + + .release-main { + padding-right: 16px; + } + + .release-side { + width: 100%; + padding: 0 10px 10px 16px; + justify-content: flex-start; + } + + .release-size { + width: auto; + margin-left: auto; + } + + kbd { + display: none; + } +} diff --git a/public/app.js b/public/app.js new file mode 100644 index 0000000..ab2e118 --- /dev/null +++ b/public/app.js @@ -0,0 +1,436 @@ +// Usenet Indexer web interface. No framework; all Usenet-derived text is +// rendered with textContent, never as HTML. + +const $ = (selector) => document.querySelector(selector) +const FILTERS = ['readable', 'complete', 'hide_encrypted', 'par2_only'] +const SOURCE_LABELS = { + stem: 'Filename', + subject: 'Subject', + post: 'Post id', + par2: 'par2 contents', + rar: 'RAR headers', + 'weak-stem': 'Obfuscated filename', + nzb: 'Posted NZB', + spot: 'Spotnet', +} + +const state = { + q: '', + sort: 'newest', + source: '', + poster: '', + readable: false, + complete: false, + hide_encrypted: false, + par2_only: false, + page: 1, + release: null, +} +let requestId = 0 + +// --- Formatting ------------------------------------------------------------ + +function formatSize(bytes) { + if (!bytes) return '–' + const units = ['B', 'KB', 'MB', 'GB', 'TB'] + let value = bytes + let unit = 0 + while (value >= 1000 && unit < units.length - 1) { + value /= 1000 + unit++ + } + return `${value >= 100 || unit === 0 ? Math.round(value) : value.toFixed(1)} ${units[unit]}` +} + +const number = new Intl.NumberFormat() + +function formatAge(date) { + if (!date) return '' + const seconds = (Date.now() - new Date(date).getTime()) / 1000 + if (seconds < 90) return 'just now' + const steps = [ + [60, 'minute'], + [3600, 'hour'], + [86400, 'day'], + [2592000, 'month'], + [31536000, 'year'], + ] + let [size, name] = steps[0] + for (const step of steps) if (seconds >= step[0]) [size, name] = step + const n = Math.floor(seconds / size) + return `${n} ${name}${n === 1 ? '' : 's'} ago` +} + +function el(tag, props = {}, ...children) { + const node = document.createElement(tag) + for (const [key, value] of Object.entries(props)) { + if (value === undefined || value === null || value === false) continue + if (key === 'class') node.className = value + else if (key === 'text') node.textContent = value + else if (key.startsWith('on')) node.addEventListener(key.slice(2), value) + else node.setAttribute(key, value === true ? '' : value) + } + for (const child of children.flat()) { + if (child === null || child === undefined || child === false) continue + node.append(child instanceof Node ? child : document.createTextNode(String(child))) + } + return node +} + +function badge(text, tone = '', title) { + return el('span', { class: `badge${tone ? ` badge-${tone}` : ''}`, text, title }) +} + +// --- URL state ------------------------------------------------------------- + +function readUrl() { + const params = new URLSearchParams(location.search) + state.q = params.get('q') ?? '' + state.sort = params.get('sort') ?? 'newest' + state.source = params.get('source') ?? '' + state.poster = params.get('poster') ?? '' + for (const f of FILTERS) state[f] = params.get(f) === '1' + state.release = params.get('release') ? Number(params.get('release')) : null +} + +function writeUrl(push = false) { + const params = new URLSearchParams() + if (state.q) params.set('q', state.q) + if (state.sort !== 'newest') params.set('sort', state.sort) + if (state.source) params.set('source', state.source) + if (state.poster) params.set('poster', state.poster) + for (const f of FILTERS) if (state[f]) params.set(f, '1') + if (state.release) params.set('release', String(state.release)) + const url = `${location.pathname}${params.size ? `?${params}` : ''}` + history[push ? 'pushState' : 'replaceState'](null, '', url) +} + +function syncControls() { + $('#q').value = state.q + $('#sort').value = state.sort + $('#source').value = state.source + for (const f of FILTERS) { + document.querySelector(`[data-filter="${f}"]`).setAttribute('aria-pressed', String(state[f])) + } + $('#poster-filter').hidden = !state.poster + $('#poster-filter-label').textContent = `Poster: ${state.poster}` +} + +// --- API ------------------------------------------------------------------- + +async function api(path) { + const response = await fetch(path, { headers: { accept: 'application/json' } }) + if (!response.ok) throw new Error(`${response.status} ${response.statusText}`) + return response.json() +} + +function searchParams(page) { + const params = new URLSearchParams({ sort: state.sort, page: String(page), per_page: '50' }) + if (state.q) params.set('q', state.q) + if (state.source) params.set('source', state.source) + if (state.poster) params.set('poster', state.poster) + for (const f of FILTERS) if (state[f]) params.set(f, '1') + return params +} + +// --- Stats ----------------------------------------------------------------- + +async function loadStats() { + try { + const stats = await api('/api/v1/stats') + const readable = stats.releases ? Math.round((100 * stats.readableReleases) / stats.releases) : 0 + const items = [ + ['Releases', number.format(stats.releases)], + ['Readable', `${readable}%`], + ['Files', number.format(stats.files)], + ['Size', formatSize(stats.size)], + ['Indexed', formatAge(stats.lastIndexedAt) || '–'], + ] + $('#stats').replaceChildren(...items.map(([dt, dd]) => el('div', {}, el('dt', { text: dt }), el('dd', { text: dd })))) + } catch { + // Stats are decoration; the page works without them. + } +} + +// --- Results --------------------------------------------------------------- + +function releaseRow(release) { + const row = $('#row-template').content.firstElementChild.cloneNode(true) + row.dataset.id = release.id + if (release.id === state.release) row.classList.add('selected') + + const name = row.querySelector('.release-name') + name.textContent = release.name + if (!release.readable) name.classList.add('obfuscated') + + const meta = [ + formatAge(release.postedAt), + release.poster ? release.poster.replace(/\s*<.*>$/, '') : 'obfuscated poster', + release.groups?.length ? release.groups[0] + (release.groups.length > 1 ? ` +${release.groups.length - 1}` : '') : '', + ].filter(Boolean) + row.querySelector('.release-meta').textContent = meta.join(' · ') + + const badges = row.querySelector('.badges') + if (release.par2Only) badges.append(badge('par2 only', 'warn', 'Only the par2 files arrived; the data they describe is missing')) + if (release.container === 'nzbe') badges.append(badge('Encrypted NZB', 'bad', 'An encrypted NZB container: unusable without its key')) + else if (release.container === 'rclone') badges.append(badge('Encrypted (rclone)', 'bad', 'rclone crypt: names and contents encrypted, unusable without the key')) + else if (release.encrypted) badges.append(badge('Password', 'bad', 'Password-protected archive')) + const files = `${release.fileCount} ${release.fileCount === 1 ? 'file' : 'files'}` + if (release.complete) badges.append(badge(files, 'good', 'Complete')) + else if (release.fileTotal) badges.append(badge(`${release.fileCount}/${release.fileTotal}`, 'warn', 'Files present / announced')) + else if (release.listedFiles) { + badges.append(badge(`${release.listedPresent}/${release.listedFiles} listed`, 'warn', 'Files the par2 set lists: present / total')) + } else badges.append(badge(files)) + if (release.source && release.source !== 'stem' && release.source !== 'subject') { + badges.append(badge(SOURCE_LABELS[release.source] ?? release.source, 'accent', 'How this release was recognised')) + } + + row.querySelector('.release-size').textContent = formatSize(release.size) + const nzb = row.querySelector('.nzb') + nzb.href = `/api/v1/releases/${release.id}/nzb` + row.querySelector('.release-main').addEventListener('click', () => openRelease(release.id, true)) + return row +} + +async function loadResults({ append = false } = {}) { + const id = ++requestId + if (!append) state.page = 1 + $('#more').hidden = true + if (!append) $('#summary').textContent = 'Searching…' + + try { + const result = await api(`/api/v1/releases?${searchParams(state.page)}`) + if (id !== requestId) return // a newer search superseded this one + const rows = result.releases.map(releaseRow) + if (append) $('#list').append(...rows) + else $('#list').replaceChildren(...rows) + + const shown = $('#list').children.length + $('#summary').textContent = + result.total === 0 ? '' : `${number.format(result.total)} ${result.total === 1 ? 'release' : 'releases'}` + + (shown < result.total ? ` · showing ${number.format(shown)}` : '') + $('#empty').hidden = result.total !== 0 + $('#more').hidden = shown >= result.total + } catch (error) { + if (id !== requestId) return + $('#summary').textContent = `Search failed: ${error.message}` + } +} + +let debounce +function scheduleSearch() { + clearTimeout(debounce) + debounce = setTimeout(() => { + writeUrl() + loadResults() + }, 220) +} + +// --- Detail drawer --------------------------------------------------------- + +function fact(label, value) { + if (value === null || value === undefined || value === '') return null + return el('div', {}, el('dt', { text: label }), el('dd', {}, value)) +} + +function section(title, note, content) { + return [el('h3', { class: 'section-title' }, title, note ? el('small', { text: note }) : null), content] +} + +function renderDetail(release) { + $('#detail-name').textContent = release.name + $('#detail-source').textContent = `Recognised by ${SOURCE_LABELS[release.source] ?? release.source ?? 'filename'}` + + const actions = el( + 'div', + { class: 'actions' }, + el('a', { class: 'button button-primary', href: `/api/v1/releases/${release.id}/nzb` }, + el('span', { text: 'Download NZB' })), + el('button', { + class: 'button', + type: 'button', + text: 'Copy name', + onclick: async (event) => { + await navigator.clipboard?.writeText(release.name) + event.target.textContent = 'Copied' + setTimeout(() => (event.target.textContent = 'Copy name'), 1200) + }, + }) + ) + + const poster = release.poster + ? el('button', { + class: 'link-button', + type: 'button', + text: release.poster.replace(/\s*<.*>$/, ''), + title: 'Show releases by this poster', + onclick: () => { + state.poster = release.poster + closeRelease() + syncControls() + writeUrl(true) + loadResults() + }, + }) + : 'Random per article (obfuscated)' + + const listedPresent = release.par2Files.filter((f) => f.present).length + const facts = el( + 'dl', + { class: 'facts' }, + fact('Size', formatSize(release.size)), + fact('Files', release.fileTotal ? `${release.fileCount} of ${release.fileTotal}` : `${release.fileCount} posted`), + fact('par2 lists', release.par2Files.length ? `${listedPresent} of ${release.par2Files.length} present` : null), + fact('Posted', release.postedAt ? `${new Date(release.postedAt).toLocaleString()} (${formatAge(release.postedAt)})` : null), + fact('Poster', poster), + fact('Groups', release.groups?.join(', ')) + ) + + const notices = [] + if (release.container === 'nzbe') { + notices.push(el('div', { class: 'notice notice-bad', text: 'Encrypted NZB container ("nzbe"): an encrypted download index for others, unreadable without its key.' })) + } else if (release.container === 'rclone') { + notices.push(el('div', { class: 'notice notice-bad', text: 'rclone crypt: file names and contents are encrypted with a password and salt that are not part of the post.' })) + } else if (release.encrypted) { + notices.push(el('div', { class: 'notice notice-bad', text: 'Password-protected: the archive needs a password that is not part of the post.' })) + } + if (release.pending.binaries > 0) { + const pct = release.pending.partsTotal ? Math.round((100 * release.pending.partsHave) / release.pending.partsTotal) : 0 + notices.push( + el('div', { class: 'notice notice-warn' }, + el('div', { style: 'flex:1' }, + `${release.pending.binaries} more ${release.pending.binaries === 1 ? 'file is' : 'files are'} still arriving (${pct}% of their parts seen).`, + el('div', { class: 'progress' }, el('span', { style: `width:${pct}%` })))) + ) + } + + const files = el('table', { class: 'file-table' }, + el('tbody', {}, release.files.map((f) => + el('tr', {}, + el('td', { class: 'name' }, f.name, + f.realName && f.realName !== f.postedName ? el('span', { class: 'posted', text: `posted as ${f.postedName}` }) : null), + el('td', { class: 'num', text: formatSize(f.size) }), + el('td', { class: 'num', text: `${f.parts} ${f.parts === 1 ? 'part' : 'parts'}` }))))) + + const par2 = release.par2Files.length + ? el('table', { class: 'file-table' }, + el('tbody', {}, release.par2Files.map((f) => + el('tr', {}, + el('td', { class: `state ${f.present ? 'ok' : 'missing'}`, text: f.present ? '✓' : '·', title: f.present ? 'Present' : 'Not seen yet' }), + el('td', { class: 'name', text: f.name }), + el('td', { class: 'num', text: formatSize(f.size) }))))) + : null + + const contents = release.contents.length + ? el('table', { class: 'file-table' }, + el('tbody', {}, release.contents.map((entry) => + el('tr', {}, + el('td', { class: 'name', text: entry.name + (entry.directory ? '/' : '') }), + el('td', { class: 'num', text: entry.size === null ? '' : formatSize(entry.size) }), + el('td', { class: 'state', text: entry.encrypted ? '🔒' : '', title: entry.encrypted ? 'Encrypted' : undefined }))))) + : null + + $('#detail').replaceChildren( + actions, + ...notices, + facts, + ...section('Files', `${release.files.length} posted`, files), + ...(par2 ? section('par2 file list', `${listedPresent} of ${release.par2Files.length} present`, par2) : []), + ...(contents ? section('Archive contents', `${release.contents.length} entries`, contents) : []) + ) +} + +async function openRelease(id, push = false) { + state.release = id + writeUrl(push) + document.querySelectorAll('.release.selected').forEach((r) => r.classList.remove('selected')) + document.querySelector(`.release[data-id="${id}"]`)?.classList.add('selected') + + $('#drawer').hidden = false + $('#scrim').hidden = false + $('#detail-name').textContent = '' + $('#detail-source').textContent = '' + $('#detail').replaceChildren(...Array.from({ length: 6 }, () => el('div', { class: 'skeleton' }))) + $('#close').focus() + try { + renderDetail(await api(`/api/v1/releases/${id}`)) + } catch (error) { + $('#detail').replaceChildren(el('p', { class: 'empty', text: `Could not load this release (${error.message}).` })) + } +} + +function closeRelease(push = false) { + if (!state.release) return + const id = state.release + state.release = null + $('#drawer').hidden = true + $('#scrim').hidden = true + writeUrl(push) + document.querySelector(`.release[data-id="${id}"] .release-main`)?.focus() +} + +// --- Wiring ---------------------------------------------------------------- + +$('#q').addEventListener('input', (event) => { + state.q = event.target.value.trim() + scheduleSearch() +}) +$('#sort').addEventListener('change', (event) => { + state.sort = event.target.value + writeUrl() + loadResults() +}) +$('#source').addEventListener('change', (event) => { + state.source = event.target.value + writeUrl() + loadResults() +}) +for (const button of document.querySelectorAll('[data-filter]')) { + button.addEventListener('click', () => { + const f = button.dataset.filter + state[f] = !state[f] + button.setAttribute('aria-pressed', String(state[f])) + writeUrl() + loadResults() + }) +} +$('#poster-clear').addEventListener('click', () => { + state.poster = '' + syncControls() + writeUrl() + loadResults() +}) +$('#more').addEventListener('click', () => { + state.page++ + loadResults({ append: true }) +}) +$('#close').addEventListener('click', () => closeRelease(true)) +$('#scrim').addEventListener('click', () => closeRelease(true)) + +document.addEventListener('keydown', (event) => { + if (event.key === 'Escape') closeRelease(true) + if (event.key === '/' && document.activeElement !== $('#q')) { + event.preventDefault() + $('#q').focus() + $('#q').select() + } +}) + +window.addEventListener('popstate', () => { + const previous = { ...state } + readUrl() + syncControls() + const searchChanged = ['q', 'sort', 'source', 'poster', ...FILTERS].some((k) => previous[k] !== state[k]) + if (searchChanged) loadResults() + if (state.release && state.release !== previous.release) openRelease(state.release) + if (!state.release && previous.release) { + $('#drawer').hidden = true + $('#scrim').hidden = true + } +}) + +readUrl() +syncControls() +loadStats() +loadResults() +if (state.release) openRelease(state.release) diff --git a/public/index.html b/public/index.html new file mode 100644 index 0000000..9e9415d --- /dev/null +++ b/public/index.html @@ -0,0 +1,107 @@ + + + + + + Usenet Indexer + + + + +
+ + + Usenet Indexer + +
+
+ +
+
+
+ + +
+
+ + + + + + +
+
+ +
+

+
    +
    + +
    + +
    +
    + + + + + + + + + diff --git a/start/env.ts b/start/env.ts index 635bf29..d5f0bb4 100644 --- a/start/env.ts +++ b/start/env.ts @@ -32,6 +32,8 @@ export default await Env.create(new URL('../', import.meta.url), { DB_USER: Env.schema.string(), DB_PASSWORD: Env.schema.secret.optional(), DB_DATABASE: Env.schema.string(), + DB_POOL_MAX: Env.schema.number.optional(), + DB_SYNCHRONOUS_COMMIT: Env.schema.enum.optional(['on', 'off'] as const), // Redis REDIS_HOST: Env.schema.string({ format: 'host' }), @@ -44,4 +46,32 @@ export default await Env.create(new URL('../', import.meta.url), { NNTP_USER: Env.schema.string(), NNTP_PASSWORD: Env.schema.secret(), NNTP_SECURE: Env.schema.boolean(), + NNTP_POOL_SIZE: Env.schema.number.optional(), + + // Indexer (config/indexer.ts) + INDEX_INTERVAL_MS: Env.schema.number.optional(), + INDEX_BATCH_SIZE: Env.schema.number.optional(), + INDEX_MAX_BATCHES_PER_TICK: Env.schema.number.optional(), + INDEX_INITIAL_BACKFILL: Env.schema.number.optional(), + FETCH_CONCURRENCY: Env.schema.number.optional(), + BODY_CONCURRENCY: Env.schema.number.optional(), + COLLECTION_CONCURRENCY: Env.schema.number.optional(), + DEFERRED_SETTLE_MS: Env.schema.number.optional(), + DEFERRED_EXPAND_MAX_PARTS: Env.schema.number.optional(), + DEFERRED_CONCURRENCY: Env.schema.number.optional(), + SPOT_CONCURRENCY: Env.schema.number.optional(), + + // Newznab API (/api): required as ?apikey= when set + NEWZNAB_API_KEY: Env.schema.secret.optional(), + + // Garbage collection (config/gc.ts) + GC_INTERVAL_MS: Env.schema.number.optional(), + GC_COMPLETED_AGE_SECONDS: Env.schema.number.optional(), + GC_FAILED_AGE_SECONDS: Env.schema.number.optional(), + GC_BINARY_STALE_SECONDS: Env.schema.number.optional(), + GC_BODY_QUEUE_MAX: Env.schema.number.optional(), + GC_DEFERRED_RETENTION_DAYS: Env.schema.number.optional(), + GC_SINGLE_ARTICLE_SECONDS: Env.schema.number.optional(), + GC_EMPTY_RELEASE_SECONDS: Env.schema.number.optional(), + GC_DELETE_BUDGET_MS: Env.schema.number.optional(), }) diff --git a/start/kernel.ts b/start/kernel.ts index d3c18be..5514e48 100644 --- a/start/kernel.ts +++ b/start/kernel.ts @@ -26,6 +26,7 @@ server.use([ () => import('#middleware/force_json_response_middleware'), () => import('#middleware/container_bindings_middleware'), () => import('@adonisjs/cors/cors_middleware'), + () => import('@adonisjs/static/static_middleware') ]) /** diff --git a/start/routes.ts b/start/routes.ts index 29fc17f..762552d 100644 --- a/start/routes.ts +++ b/start/routes.ts @@ -11,9 +11,21 @@ import { middleware } from '#start/kernel' import router from '@adonisjs/core/services/router' import { controllers } from '#generated/controllers' -router.get('/', () => { - return { hello: 'world' } -}) +const ReleasesController = () => import('#controllers/releases_controller') +const NewznabController = () => import('#controllers/newznab_controller') + +// Newznab API for Prowlarr / Sonarr / Radarr ("Generic Newznab", API path /api). +router.get('/api', [NewznabController, 'handle']) + +// Web interface: public/index.html is served by the static middleware at "/". +router + .group(() => { + router.get('releases', [ReleasesController, 'index']) + router.get('releases/:id', [ReleasesController, 'show']).where('id', router.matchers.number()) + router.get('releases/:id/nzb', [ReleasesController, 'nzb']).where('id', router.matchers.number()) + router.get('stats', [ReleasesController, 'stats']) + }) + .prefix('/api/v1') router .group(() => { diff --git a/tests/functional/deferred.spec.ts b/tests/functional/deferred.spec.ts new file mode 100644 index 0000000..f91af6e --- /dev/null +++ b/tests/functional/deferred.spec.ts @@ -0,0 +1,67 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import DeferredService, { deferredName, isDeferred } from '#services/DeferredService' + +const header = (id: string, name: string, extra: Record = {}) => ({ + 'message-id': `<${id.padEnd(32, '0')}@ngPost>`, + 'subject': `"${name}" yEnc`, + 'from': 'pKeUimunULJZz@ngPost.com', + ':bytes': '740000', + 'date': 'Mon, 28 Sep 2026 10:00:00 +0000', + ...extra, +}) + +test.group('DeferredService', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('recognises deferred uploads by message-id and a quoted name', ({ assert }) => { + assert.equal(deferredName('[026/109] "HFIX7fbQAuH4PZsvGMw7Pz.part025.rar" yEnc'), 'HFIX7fbQAuH4PZsvGMw7Pz.part025.rar') + assert.isNull(deferredName('no quotes here')) + assert.isTrue(isDeferred(header('ab', 'x.part1.rar'))) + assert.isFalse(isDeferred({ ...header('ab', 'x'), 'message-id': '' })) + }) + + test('articles without a name in the subject are kept as article ranges', async ({ assert }) => { + const hex = (id: string, n: number) => ({ articleNumber: n, header: { ...header(id, 'x'), subject: id.padEnd(32, 'f') } }) + const files = await DeferredService.record('alt.binaries.test', [hex('a', 100), hex('b', 180), hex('c', 90_000), hex('d', 150)]) + assert.equal(files, 0) + const ranges = await db.from('deferred_ranges').orderBy('first_article') + assert.deepEqual( + ranges.map((r) => [Number(r.first_article), Number(r.last_article), r.article_count]), + [[100, 180, 3], [90_000, 90_000, 1]] + ) + assert.lengthOf(await db.from('deferred_files'), 0) + }) + + test('aggregates articles per file, across batches, counting only the first group', async ({ assert }) => { + await DeferredService.record('alt.binaries.test', [ + { articleNumber: 105, header: header('b', 'a.part1.rar') }, + { articleNumber: 101, header: header('a', 'a.part1.rar') }, + { articleNumber: 102, header: header('c', 'b.par2', { ':bytes': '1000' }) }, + ]) + await DeferredService.record('alt.binaries.test', [{ articleNumber: 110, header: header('d', 'a.part1.rar') }]) + // A crosspost seen in another group doesn't count twice. + await DeferredService.record('alt.binaries.misc', [{ articleNumber: 9, header: header('a', 'a.part1.rar') }]) + + const rows = await db.from('deferred_files').orderBy('name') + assert.lengthOf(rows, 2) + const [a, b] = rows + assert.include(a, { name: 'a.part1.rar', group_name: 'alt.binaries.test', article_count: 3, status: 'pending' }) + assert.equal(Number(a.first_article), 101) + assert.equal(Number(a.last_article), 110) + assert.equal(a.first_message_id, 'a'.padEnd(32, '0') + '@ngPost') + assert.equal(Number(a.bytes), 3 * 740000) + assert.include(b, { name: 'b.par2', article_count: 1 }) + }) + + test('only claims files that stopped growing', async ({ assert }) => { + await DeferredService.record('alt.binaries.test', [{ articleNumber: 1, header: header('a', 'new.rar') }]) + assert.lengthOf(await DeferredService.claim(10), 0) + await db.from('deferred_files').update({ last_seen_at: new Date(Date.now() - 24 * 60 * 60 * 1000) }) + const claimed = await DeferredService.claim(10) + assert.lengthOf(claimed, 1) + assert.equal(claimed[0].status, 'processing') + assert.lengthOf(await DeferredService.claim(10), 0) + }) +}) diff --git a/tests/functional/fetch_ranges.spec.ts b/tests/functional/fetch_ranges.spec.ts new file mode 100644 index 0000000..a22e067 --- /dev/null +++ b/tests/functional/fetch_ranges.spec.ts @@ -0,0 +1,46 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import FetchRangeService from '#services/FetchRangeService' + +async function groupWithRanges(ranges: [number, number, string][]) { + const now = new Date() + const [group] = await db + .table('groups') + .insert({ name: `test.ranges.${Date.now()}`, active: false, last_indexed_id: 100, last_queued_id: 700, created_at: now, updated_at: now }) + .returning(['id', 'name']) + for (const [start, end, status] of ranges) { + await db.table('fetch_ranges').insert({ group_id: group.id, start_id: start, end_id: end, status, created_at: now, updated_at: now }) + } + return { id: Number(group.id), name: group.name } +} + +test.group('FetchRangeService.advanceWatermark', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('stops just below the first range that is not done', async ({ assert }) => { + const g = await groupWithRanges([ + [101, 200, 'done'], + [201, 300, 'failed'], + [301, 400, 'done'], + [401, 700, 'queued'], + ]) + assert.equal(await FetchRangeService.advanceWatermark(g), 200n) + // done ranges are forgotten, the gap and the queued range stay + const left = await db.from('fetch_ranges').where('group_id', g.id).orderBy('start_id').select('status') + assert.deepEqual(left.map((r) => r.status), ['failed', 'queued']) + }) + + test('reaches the queue cursor when everything is done', async ({ assert }) => { + const g = await groupWithRanges([ + [101, 400, 'done'], + [401, 700, 'done'], + ]) + assert.equal(await FetchRangeService.advanceWatermark(g), 700n) + }) + + test('never moves backwards', async ({ assert }) => { + const g = await groupWithRanges([[50, 60, 'failed']]) + assert.equal(await FetchRangeService.advanceWatermark(g), 100n) + }) +}) diff --git a/tests/functional/ingest.spec.ts b/tests/functional/ingest.spec.ts new file mode 100644 index 0000000..c7fccf3 --- /dev/null +++ b/tests/functional/ingest.spec.ts @@ -0,0 +1,148 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import IngestService, { type IngestSegment } from '#services/IngestService' +import QueueService from '#services/QueueService' +import { fileKey } from '#lib/subject/index' + +function segment(overrides: Partial): IngestSegment { + const poster = 'ingest-test@example.invalid' + const subject = '"test-file.bin" yEnc' + return { + fileKey: fileKey(poster, subject, 3), + poster, + subject, + name: 'test-file.bin', + totalParts: 3, + number: 1, + messageId: 'part1@test', + bytes: 100, + groups: ['alt.binaries.test'], + ...overrides, + } +} + +async function binaryFor(key: string) { + return db.from('binaries').where('file_key', key).first() +} + +// None of these cases complete a binary, so nothing is queued in Redis. +test.group('IngestService', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + // IngestService opens the BullMQ queues; close them so the test process can exit. + group.teardown(() => QueueService.closeAll()) + + test('existing binaries are only rewritten when something changes', async ({ assert }) => { + const first = segment({ number: 1, postedAt: new Date('2026-09-28T12:00:00Z') }) + await IngestService.ingest([first]) + const before = await binaryFor(first.fileKey) + + // A later part within TOUCH_MS: nothing to change, so the row is left alone. + await IngestService.ingest([segment({ number: 2, messageId: 'part2@test', postedAt: new Date('2026-09-28T12:05:00Z') })]) + const same = await binaryFor(first.fileKey) + assert.equal(+same.last_seen_at, +before.last_seen_at) + assert.equal(await db.from('parts').where('binary_id', before.id).count('* as n').then((r) => Number(r[0].n)), 2) + + // An earlier post date and a stale last_seen_at are merged. + await db.from('binaries').where('id', before.id).update({ last_seen_at: new Date(Date.now() - 60 * 60 * 1000) }) + await IngestService.ingest([segment({ number: 1, postedAt: new Date('2026-09-28T11:00:00Z') })]) + const after = await binaryFor(first.fileKey) + assert.equal(after.posted_at.toISOString(), '2026-09-28T11:00:00.000Z') + assert.isAbove(+after.last_seen_at, Date.now() - 60_000) + }) + + test('ingesting the same segments twice is a no-op', async ({ assert }) => { + const segments = [segment({ number: 1 }), segment({ number: 2, messageId: 'part2@test' })] + await IngestService.ingest(segments) + await IngestService.ingest(segments) + + const binary = await binaryFor(segments[0].fileKey) + const [{ count }] = await db.from('parts').where('binary_id', binary.id).count('* as count') + assert.equal(Number(count), 2) + assert.isNull(binary.completed_at) + assert.isNull(binary.conflicted_at) + }) + + test('a weak identity with two articles for one part number is flagged and never completes', async ({ + assert, + }) => { + const weak = { weakIdentity: true } + await IngestService.ingest([segment({ ...weak, number: 1, messageId: 'volume-a-1@test' })]) + // Same name, poster and size, but another file: the second volume of a set. + await IngestService.ingest([ + segment({ ...weak, number: 1, messageId: 'volume-b-1@test' }), + segment({ ...weak, number: 2, messageId: 'volume-b-2@test' }), + segment({ ...weak, number: 3, messageId: 'volume-b-3@test' }), + ]) + + const binary = await binaryFor(segment({}).fileKey) + assert.isNotNull(binary.conflicted_at) + assert.isNull(binary.completed_at, 'all part numbers are present, but it must not complete') + }) + + test('a conflict inside a single batch is detected too', async ({ assert }) => { + await IngestService.ingest([ + segment({ weakIdentity: true, number: 2, messageId: 'a@test' }), + segment({ weakIdentity: true, number: 2, messageId: 'b@test' }), + ]) + assert.isNotNull((await binaryFor(segment({}).fileKey)).conflicted_at) + }) + + test('on the subject path a second article for a part is a repost, not a conflict', async ({ + assert, + }) => { + await IngestService.ingest([segment({ number: 1, messageId: 'original@test' })]) + await IngestService.ingest([segment({ number: 1, messageId: 'repost@test' })]) + + const binary = await binaryFor(segment({}).fileKey) + assert.isNull(binary.conflicted_at) + const part = await db.from('parts').where({ binary_id: binary.id, number: 1 }).first() + assert.equal(part.message_id, 'original@test') + }) + + test('an obfuscated file joins a known par2 set of exactly its size', async ({ assert }) => { + const now = new Date() + const [release] = await db + .table('releases') + .insert({ release_key: 'k'.repeat(40), name: 'Some.Show.S01E01', source: 'par2', groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now }) + .returning('id') + await db.table('release_files').insert({ release_id: release.id, par2_file_id: 'f'.repeat(32), name: 'Some.Show.S01E01.mkv', size: 5467240532, md5_16k: '0'.repeat(32) }) + + const weak = { weakIdentity: true, postedAt: now, release: null } + await IngestService.ingest([ + segment({ ...weak, fileKey: 'a'.repeat(40), size: 5467240532, totalParts: 7628, messageId: 'match@test' }), + segment({ ...weak, fileKey: 'b'.repeat(40), size: 5467240533, totalParts: 7628, messageId: 'other@test' }), + ]) + + const matched = await binaryFor('a'.repeat(40)) + const other = await binaryFor('b'.repeat(40)) + assert.equal(Number(matched.release_id), Number(release.id)) + assert.isNull(other.release_id) + }) + + test('an identical repost (same part CRC) is not a conflict', async ({ assert }) => { + const weak = { weakIdentity: true, pcrc32: 'abcd1234', partSize: 700 } + await IngestService.ingest([segment({ ...weak, number: 1, messageId: 'first@test' })]) + await IngestService.ingest([segment({ ...weak, number: 1, messageId: 'repost@test' })]) + const binary = await binaryFor(segment({}).fileKey) + assert.isNull(binary.conflicted_at) + + await IngestService.ingest([segment({ ...weak, pcrc32: 'ffff0000', number: 1, messageId: 'other@test' })]) + assert.isNotNull((await binaryFor(segment({}).fileKey)).conflicted_at) + }) + + test('a release takes its file total from whichever file carries the counter', async ({ assert }) => { + const { releaseIdentity } = await import('#lib/release/index') + const poster = 'archiver@example.invalid' + const file = (subject: string, name: string, key: string) => + segment({ fileKey: key.padEnd(40, 'k'), poster, subject, name, totalParts: 1, messageId: `${key}@test`, + release: releaseIdentity({ poster, subject, name, weak: false }) }) + // The counter-less par2 index comes first in the batch. + await IngestService.ingest([ + file('"x.tar.zst.par2" yEnc 3100', 'x.tar.zst.par2', 'index'), + file('[1/2] - "x.tar.zst" yEnc 1000', 'x.tar.zst', 'archive'), + ]) + const release = await db.from('releases').where('name', 'x').where('poster', poster).first() + assert.equal(release.file_total, 2) + }) +}) diff --git a/tests/functional/names.spec.ts b/tests/functional/names.spec.ts new file mode 100644 index 0000000..0ecabba --- /dev/null +++ b/tests/functional/names.spec.ts @@ -0,0 +1,94 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import NameService from '#services/NameService' + +async function release(name: string, files: { name: string; size: number; md5_16k: string }[]) { + const now = new Date() + const [row] = await db + .table('releases') + .insert({ release_key: `${name}`.padEnd(40, 'x').slice(0, 40), name, source: 'par2', groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now }) + .returning('id') + for (const [i, f] of files.entries()) { + await db.table('release_files').insert({ release_id: row.id, par2_file_id: String(i).padStart(32, '0'), name: f.name, size: f.size, md5_16k: f.md5_16k }) + } + return Number(row.id) +} + +async function binary(key: string, fields: Record) { + const now = new Date() + const [row] = await db + .table('binaries') + .insert({ file_key: key.padEnd(40, '0'), poster: 'x', subject: '"x" yEnc', name: 'Xq8r2KdLm3Zw91PqA7', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, ...fields }) + .returning('id') + return Number(row.id) +} + +test.group('NameService', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('an exact match leaves a random-named single-file release for the par2 set', async ({ assert }) => { + const season = await release('SEAL.Team.S03', [ + { name: 'SEAL.Team.S03E18.1080p.WEB-DL-nitrato.mkv', size: 939_684_767, md5_16k: 'd'.repeat(32) }, + ]) + const now = new Date() + const [weak] = await db + .table('releases') + .insert({ release_key: 'w'.repeat(40), name: 'mOzyyxCyYUjAbCdEf12', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }) + .returning('id') + const [file] = await db + .table('files') + .insert({ filename: 'mOzyyxCyYUjAbCdEf12', poster: 'x', date: now.getTime(), parts: 1, size: 1, groups: '[]', message_ids: '{}', release_id: weak.id, created_at: now, updated_at: now }) + .returning('id') + const episode = await binary('ep18', { size: 939_684_767, md5_16k: 'd'.repeat(32), release_id: weak.id, file_id: file.id }) + + await NameService.resolve([episode]) + const moved = await db.from('binaries').where('id', episode).first() + assert.equal(Number(moved.release_id), season) + assert.equal(moved.real_name, 'SEAL.Team.S03E18.1080p.WEB-DL-nitrato.mkv') + assert.equal(Number((await db.from('files').where('id', file.id).first()).release_id), season) + }) + + test('a file in a real grouping keeps it', async ({ assert }) => { + await release('Other.Set', [{ name: 'Other.Show.S01E01.mkv', size: 5_000_000, md5_16k: 'e'.repeat(32) }]) + const now = new Date() + const [nzb] = await db + .table('releases') + .insert({ release_key: 'n'.repeat(40), name: 'Qz8r2KdLm3Zw91PqA7x', source: 'nzb', groups: '[]', first_seen_at: now, last_seen_at: now }) + .returning('id') + const id = await binary('nzbfile', { size: 5_000_000, md5_16k: 'e'.repeat(32), release_id: nzb.id }) + await NameService.resolve([id]) + assert.equal(Number((await db.from('binaries').where('id', id).first()).release_id), Number(nzb.id)) + }) + + test('names equal-sized volumes exactly by their first-16k fingerprint', async ({ assert }) => { + const releaseId = await release('Some.Movie', [ + { name: 'Some.Movie.part1.rar', size: 50_000_000, md5_16k: 'a'.repeat(32) }, + { name: 'Some.Movie.part2.rar', size: 50_000_000, md5_16k: 'b'.repeat(32) }, + ]) + const one = await binary('one', { size: 50_000_000, md5_16k: 'b'.repeat(32) }) + const two = await binary('two', { size: 50_000_000, md5_16k: 'a'.repeat(32) }) + const stranger = await binary('three', { size: 50_000_000, md5_16k: 'c'.repeat(32) }) + await NameService.resolve([one, two, stranger]) + + const rows = await db.from('binaries').whereIn('id', [one, two, stranger]).orderBy('id') + assert.deepEqual(rows.map((r) => [r.real_name, r.release_id && Number(r.release_id)]), [ + ['Some.Movie.part2.rar', releaseId], + ['Some.Movie.part1.rar', releaseId], + [null, null], + ]) + }) + + test('without a fingerprint, names by size only when the size is unique in the release', async ({ assert }) => { + const releaseId = await release('Other', [ + { name: 'Other.mkv', size: 700_000_000, md5_16k: 'd'.repeat(32) }, + { name: 'Other.part1.rar', size: 10_000_000, md5_16k: 'e'.repeat(32) }, + { name: 'Other.part2.rar', size: 10_000_000, md5_16k: 'f'.repeat(32) }, + ]) + const video = await binary('video', { size: 700_000_000, release_id: releaseId }) + const volume = await binary('volume', { size: 10_000_000, release_id: releaseId }) + await NameService.resolve([video, volume]) + assert.equal((await db.from('binaries').where('id', video).first()).real_name, 'Other.mkv') + assert.isNull((await db.from('binaries').where('id', volume).first()).real_name) + }) +}) diff --git a/tests/functional/newznab.spec.ts b/tests/functional/newznab.spec.ts new file mode 100644 index 0000000..c0a9f01 --- /dev/null +++ b/tests/functional/newznab.spec.ts @@ -0,0 +1,90 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import env from '#start/env' +import testUtils from '@adonisjs/core/services/test_utils' + +let counter = 0 +async function seed(name: string, opts: { fileTotal?: number; files?: number } = {}) { + const now = new Date() + const [release] = await db + .table('releases') + .insert({ release_key: `nz${++counter}`.padEnd(40, 'n'), name, source: 'stem', poster: 'poster@example.invalid', groups: '["alt.binaries.test"]', file_total: opts.fileTotal ?? null, posted_at: now, first_seen_at: now, last_seen_at: now }) + .returning('id') + for (let i = 0; i < (opts.files ?? 1); i++) { + await db.table('files').insert({ + filename: `${name}.part${i + 1}.rar`, poster: 'poster@example.invalid', date: now.getTime(), parts: 1, size: 1_000_000, + release_id: release.id, groups: '["alt.binaries.test"]', message_ids: JSON.stringify({ '1': { id: `${name}-${i}@test`, size: 1_000_000 } }), + created_at: now, updated_at: now, + }) + } + return Number(release.id) +} + +const ids = (xml: string) => [...xml.matchAll(/(\d+)<\/guid>/g)].map((m) => Number(m[1])) + +test.group('Newznab API', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('caps lists search functions and categories', async ({ client, assert }) => { + const response = await client.get('/api').qs({ t: 'caps' }) + response.assertStatus(200) + assert.include(response.text(), '') + assert.include(response.text(), '') + assert.include(response.text(), '') + }) + + test('tvsearch matches show, season and episode, with Newznab attributes', async ({ client, assert }) => { + const e02 = await seed('Nzbtest.Show.S01E02.1080p.WEB-DL-GRP') + const e03 = await seed('Nzbtest.Show.S01E03.1080p.WEB-DL-GRP') + const movie = await seed('Nzbtest.Show.The.Movie.2020.1080p.BluRay.x264-GRP') + + const response = await client.get('/api').qs({ t: 'tvsearch', q: 'nzbtest show', season: '1', ep: '2' }) + response.assertStatus(200) + const body = response.text() + assert.deepEqual(ids(body), [e02]) + assert.include(body, '') + assert.include(body, '') + assert.include(body, '') + assert.include(body, `/api?t=get&id=${e02}`) + assert.include(body, 'type="application/x-nzb"') + + const season = await client.get('/api').qs({ t: 'tvsearch', q: 'nzbtest show', season: '1' }) + assert.sameMembers(ids(season.text()), [e02, e03]) + const movies = await client.get('/api').qs({ t: 'movie', q: 'nzbtest show' }) + assert.deepEqual(ids(movies.text()), [movie]) + }) + + test('leaves out obfuscated names and releases known to be incomplete', async ({ client, assert }) => { + const readable = await seed('Nzbtest.Other.2021.720p.WEB-DL-GRP') + const incomplete = await seed('Nzbtest.Partial.2021.720p.WEB-DL-GRP', { fileTotal: 5, files: 2 }) + const random = await seed('Xq8r2KdLm3Zw91PqA7nzbtest') + const found = ids((await client.get('/api').qs({ t: 'search', q: 'nzbtest' })).text()) + assert.include(found, readable) + assert.notInclude(found, incomplete) + assert.notInclude(found, random) + }) + + test('get returns the NZB, errors use Newznab codes', async ({ client, assert }) => { + const id = await seed('Nzbtest.Get.S02E01.720p.HDTV-GRP') + const nzb = await client.get('/api').qs({ t: 'get', id: String(id) }) + nzb.assertStatus(200) + assert.include(nzb.header('content-type'), 'application/x-nzb') + assert.include(nzb.text(), 'Nzbtest.Get.S02E01.720p.HDTV-GRP-0@test') + assert.include((await client.get('/api').qs({ t: 'get', id: '999999999' })).text(), 'code="300"') + assert.include((await client.get('/api').qs({ t: 'nonsense' })).text(), 'code="202"') + }) + + test('requires the API key when one is configured', async ({ client, assert }) => { + const previous = env.get('NEWZNAB_API_KEY') + env.set('NEWZNAB_API_KEY', 'test-key' as any) + try { + assert.include((await client.get('/api').qs({ t: 'caps' })).text(), 'code="100"') + assert.include((await client.get('/api').qs({ t: 'caps', apikey: 'test-key' })).text(), '') + const id = await seed('Nzbtest.Key.S01E01.720p.HDTV-GRP') + const search = await client.get('/api').qs({ t: 'search', q: 'nzbtest key', apikey: 'test-key' }) + assert.include(search.text(), `id=${id}&apikey=test-key`) + } finally { + env.set('NEWZNAB_API_KEY', previous as any) + } + }) +}) diff --git a/tests/functional/nzb_import.spec.ts b/tests/functional/nzb_import.spec.ts new file mode 100644 index 0000000..00a93fd --- /dev/null +++ b/tests/functional/nzb_import.spec.ts @@ -0,0 +1,97 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import NzbImportService, { nzbFileName } from '#services/NzbImportService' +import SearchService from '#services/SearchService' + +const XML = ` + + Real.Name.2026.1080p.WEB-DL-GRPhunter2 + + alt.binaries.bloaf + known-1@xknown-2@x + + + alt.binaries.cores + new-1@x + +` + +async function file(fields: Record) { + const now = new Date() + const [row] = await db + .table('files') + .insert({ poster: 'p@x', date: now.getTime(), parts: 1, size: 100, groups: '[]', created_at: now, updated_at: now, ...fields }) + .returning('*') + return row +} + +test.group('NzbImportService', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('names entries despite odd quoting', ({ assert }) => { + assert.equal(nzbFileName('Wond40 "Nachbe" Nachchbee2025bebeSD.part1.rar" yEnc (0001/1849)'), 'Nachchbee2025bebeSD.part1.rar') + assert.equal(nzbFileName('"x.par2" yEnc (1/1)'), 'x.par2') + }) + + test('joins the release of known files, adds the rest, keeps name and password', async ({ assert }) => { + const now = new Date() + const [release] = await db + .table('releases') + .insert({ release_key: 'z'.repeat(40), name: 'Xq8r2KdLm3Zw91PqA7', source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }) + .returning('id') + await file({ filename: 'Xq8r2KdLm3Zw91PqA7.part1.rar', release_id: release.id, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) }) + const posted = await file({ filename: 'Xq8r2KdLm3Zw91PqA7.nzb', message_ids: JSON.stringify({ '1': { id: 'nzb-1@x', size: 2000 } }) }) + + const result = (await NzbImportService.import(posted, XML))! + assert.equal(result.releaseId, Number(release.id)) + assert.include(result, { listed: 2, known: 1, added: 1, password: true }) + + const updated = await db.from('releases').where('id', release.id).first() + assert.equal(updated.name, 'Real.Name.2026.1080p.WEB-DL-GRP') + assert.equal(updated.password, 'hunter2') + assert.equal(updated.file_total, 2) + + const files = await db.from('files').where('release_id', release.id).orderBy('filename').select('filename', 'groups') + assert.deepEqual(files.map((f) => f.filename), ['Xq8r2KdLm3Zw91PqA7.nzb', 'Xq8r2KdLm3Zw91PqA7.part1.rar', 'Xq8r2KdLm3Zw91PqA7.part2.rar']) + + // Our own NZB passes the password on. + const nzb = await SearchService.nzb(Number(release.id)) + assert.include(nzb!.xml, 'hunter2') + + // Importing again changes nothing. + const again = (await NzbImportService.import(posted, XML))! + assert.equal(again.added, 0) + }) + + test('NZBs from one archive become separate releases named after their entries', async ({ assert }) => { + const nzbXml = (id: string) => ` + + alt.binaries.x + ${id}@x + ` + const archive = await file({ filename: 'JOCpLxSgyzOPhBWXL5riLy.rar', message_ids: JSON.stringify({ '1': { id: 'rar-1@x', size: 2000 } }) }) + + const a = (await NzbImportService.import(archive, nzbXml('a'), { entryName: 'The Kinks - Misfits-LP-1978.nzb', moveContainer: false }))! + const b = (await NzbImportService.import(archive, nzbXml('b'), { entryName: 'The Kinks - Sleepwalker-LP-1977.nzb', moveContainer: false }))! + assert.notEqual(a.releaseId, b.releaseId) + assert.equal(a.releaseName, 'The Kinks - Misfits-LP-1978') + assert.equal(b.releaseName, 'The Kinks - Sleepwalker-LP-1977') + const container = await db.from('files').where('id', archive.id).first() + assert.isNull(container.release_id) + }) + + test('a posted NZB alone in its own release moves to the release it describes', async ({ assert }) => { + const now = new Date() + const insert = async (key: string, name: string) => + (await db.table('releases').insert({ release_key: key.padEnd(40, 'r'), name, source: 'stem', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id'))[0].id + const target = await insert('target', 'Backbeat.1994.1080p.BluRay.x264-OFT') + const lonely = await insert('lonely', '.nzb') + await file({ filename: 'x.part1.rar', release_id: target, message_ids: JSON.stringify({ '1': { id: 'known-1@x', size: 700 } }) }) + const posted = await file({ filename: '.nzb', release_id: lonely, message_ids: JSON.stringify({ '1': { id: 'nzb-2@x', size: 2000 } }) }) + + await NzbImportService.import(posted, XML) + assert.equal(Number((await db.from('files').where('id', posted.id).first()).release_id), Number(target)) + assert.notExists(await db.from('releases').where('id', lonely).first()) + }) +}) diff --git a/tests/functional/par2_attach.spec.ts b/tests/functional/par2_attach.spec.ts new file mode 100644 index 0000000..1b49852 --- /dev/null +++ b/tests/functional/par2_attach.spec.ts @@ -0,0 +1,56 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import Par2Service from '#services/Par2Service' +import type { Par2Set } from '#lib/par2/index' + +const VOLUME = 10_000_952 // a standard split-volume size that recurs across posts + +async function release(key: string, name: string) { + const now = new Date() + const [row] = await db + .table('releases') + .insert({ release_key: key.padEnd(40, '0'), name, source: 'weak-stem', groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now }) + .returning('id') + return Number(row.id) +} + +async function binary(key: string, fields: Record) { + const now = new Date() + const [row] = await db + .table('binaries') + .insert({ file_key: key.padEnd(40, '0'), poster: 'x', subject: '"x" yEnc', name: 'Xq8r2KdLm3Zw91PqA7', total_parts: 14, groups: '[]', posted_at: now, first_seen_at: now, last_seen_at: now, ...fields }) + .returning(['id', 'poster', 'groups', 'posted_at']) + return row +} + +function set(id: string, files: { name: string; md5_16k: string }[]): Par2Set { + return { + recoverySetId: id.padEnd(32, '0'), + files: files.map((f, i) => ({ fileId: `${id}${i}`.padEnd(32, '0'), md5: '0'.repeat(32), md5_16k: f.md5_16k, length: VOLUME, name: f.name })), + } +} + +test.group('Par2Service.attach', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('a set does not join another post just because a size matches', async ({ assert }) => { + const other = await release('other', 'OtherPost') + await binary('other-vol', { size: VOLUME, md5_16k: 'a'.repeat(32), release_id: other }) + const par2 = await binary('par2-b', { size: 50_000, total_parts: 1 }) + + const result = await Par2Service.attach(par2, set('b', [{ name: 'PostB.part1.rar', md5_16k: 'b'.repeat(32) }])) + assert.notEqual(result.releaseId, other) + assert.equal(await db.from('release_files').where('release_id', other).count('* as n').first().then((r) => Number(r.n)), 0) + }) + + test('an exactly matching file pulls the set into its release and gets its real name', async ({ assert }) => { + const stemRelease = await release('stem', 'Xq8r2KdLm3Zw91PqA7') + const vol = await binary('vol', { size: VOLUME, md5_16k: 'c'.repeat(32), release_id: stemRelease }) + const par2 = await binary('par2-c', { size: 50_000, total_parts: 1 }) + + const result = await Par2Service.attach(par2, set('c', [{ name: 'Real.Name.part1.rar', md5_16k: 'c'.repeat(32) }])) + assert.equal(result.releaseId, stemRelease) + assert.equal((await db.from('binaries').where('id', vol.id).first()).real_name, 'Real.Name.part1.rar') + }) +}) diff --git a/tests/functional/split.spec.ts b/tests/functional/split.spec.ts new file mode 100644 index 0000000..e77bf53 --- /dev/null +++ b/tests/functional/split.spec.ts @@ -0,0 +1,84 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import IngestService, { type IngestSegment } from '#services/IngestService' +import SplitService from '#services/SplitService' +import { weakFileKey } from '#lib/subject/index' +import { crc32Hex } from '#lib/yenc/index' + +// Three 4-byte parts per volume, with real CRCs so the split volumes verify. +const DATA: Record = { A: Buffer.from('AAAAaaaa1111'), B: Buffer.from('BBBBbbbb2222') } + +const KEY = weakFileKey('a8f3Kq', 10_000_952, 3) + +function article(volume: string, number: number, articleNumber: number): IngestSegment { + return { + fileKey: KEY, + poster: `random-${articleNumber}@example.invalid`, + subject: '"a8f3Kq" yEnc', + name: `random${articleNumber}`, + totalParts: 3, + number, + messageId: `${volume}-${number}@test`, + bytes: 700_000, + articleNumber, + groups: ['alt.binaries.test'], + weakIdentity: true, + size: 10_000_952, + release: null, + pcrc32: crc32Hex(DATA[volume].subarray((number - 1) * 4, number * 4)), + partSize: 4, + fileCrc32: number === 3 ? crc32Hex(DATA[volume]) : undefined, + } +} + +test.group('SplitService', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + group.each.setup(() => { + // Keep completed volumes out of the real file queue. + const enqueue = IngestService.enqueue + IngestService.enqueue = async () => {} + return () => { + IngestService.enqueue = enqueue + } + }) + + test('splits two equal-sized volumes into verified, complete files', async ({ assert }) => { + // Volume A then volume B, arriving interleaved and out of order. + await IngestService.ingest([article('A', 1, 101), article('A', 2, 102), article('B', 1, 104)]) + await IngestService.ingest([article('B', 2, 105), article('A', 3, 103), article('B', 3, 106)]) + + const parent = await db.from('binaries').where('file_key', KEY).first() + assert.isNotNull(parent.conflicted_at) + + const result = await SplitService.splitPending() + assert.equal(result.volumes, 2) + assert.equal(result.completed, 2) + + const volumes = await db.from('binaries').where('parent_binary_id', parent.id).orderBy('id') + assert.lengthOf(volumes, 2) + for (const [i, volume] of volumes.entries()) { + assert.isNotNull(volume.completed_at) + const parts = await db.from('parts').where('binary_id', volume.id).orderBy('number').select('message_id') + const letter = i === 0 ? 'A' : 'B' + assert.deepEqual(parts.map((p) => p.message_id), [1, 2, 3].map((n) => `${letter}-${n}@test`)) + } + + // Nothing new since the split: the next run leaves it alone. + assert.equal((await SplitService.splitPending()).parents, 0) + }) + + test('a mix that fails CRC verification is not turned into a file', async ({ assert }) => { + // B's last part is missing: the only complete candidate would mix volumes. + await IngestService.ingest([article('A', 1, 101), article('B', 1, 102), article('A', 2, 103)]) + await IngestService.ingest([article('B', 2, 104), article('A', 3, 105)]) + const parent = await db.from('binaries').where('file_key', KEY).first() + const result = await SplitService.splitPending() + const volumes = await db.from('binaries').where('parent_binary_id', parent.id) + for (const volume of volumes) { + const parts = await db.from('parts').where('binary_id', volume.id).select('message_id') + assert.lengthOf(new Set(parts.map((p) => p.message_id[0])), 1) + } + assert.isAtMost(result.completed, 1) + }) +}) diff --git a/tests/functional/web_api.spec.ts b/tests/functional/web_api.spec.ts new file mode 100644 index 0000000..c4d60ba --- /dev/null +++ b/tests/functional/web_api.spec.ts @@ -0,0 +1,106 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' + +async function seed() { + const now = new Date() + const [release] = await db + .table('releases') + .insert({ release_key: 'w'.repeat(40), name: 'Webtest.Show.S01E02.1080p', source: 'par2', groups: '["alt.binaries.test"]', posted_at: now, first_seen_at: now, last_seen_at: now, contents: JSON.stringify([{ name: 'Webtest.Show.S01E02.1080p/episode.mkv', size: 1000 }]) }) + .returning('id') + await db.table('files').insert({ + filename: 'Xq8r2KdLm3Zw91PqA7webtest', real_name: 'Webtest.Show.S01E02.1080p.mkv', poster: 'p@example.invalid', + date: now.getTime(), parts: 2, size: 1400, release_id: release.id, groups: '["alt.binaries.test"]', + message_ids: JSON.stringify({ '1': { id: 'one@test', size: 700 }, '2': { id: 'two@test', size: 700 } }), + created_at: now, updated_at: now, + }) + return Number(release.id) +} + +test.group('Web API', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('searches by words in any order, with dots and spaces interchangeable', async ({ client, assert }) => { + const id = await seed() + for (const q of ['webtest show s01e02', 'Webtest.Show', 'episode.mkv 1080p']) { + const response = await client.get('/api/v1/releases').qs({ q }) + response.assertStatus(200) + assert.include(response.body().releases.map((r: any) => r.id), id, q) + } + const none = await client.get('/api/v1/releases').qs({ q: 'webtest nosuchword' }) + assert.notInclude(none.body().releases.map((r: any) => r.id), id) + }) + + test('filters and details', async ({ client, assert }) => { + const id = await seed() + const readable = await client.get('/api/v1/releases').qs({ q: 'webtest', readable: '1', source: 'par2' }) + assert.include(readable.body().releases.map((r: any) => r.id), id) + const other = await client.get('/api/v1/releases').qs({ q: 'webtest', source: 'rar' }) + assert.notInclude(other.body().releases.map((r: any) => r.id), id) + + const detail = await client.get(`/api/v1/releases/${id}`) + detail.assertStatus(200) + assert.equal(detail.body().files[0].name, 'Webtest.Show.S01E02.1080p.mkv') + assert.equal(detail.body().fileCount, 1) + assert.equal(detail.body().size, 1400) + ;(await client.get('/api/v1/releases/999999999')).assertStatus(404) + }) + + test('serves an NZB with the real file name', async ({ client, assert }) => { + const id = await seed() + const response = await client.get(`/api/v1/releases/${id}/nzb`) + response.assertStatus(200) + assert.include(response.header('content-type'), 'application/x-nzb') + assert.include(response.header('content-disposition'), 'Webtest.Show.S01E02.1080p.nzb') + assert.include(response.text(), '"Webtest.Show.S01E02.1080p.mkv" yEnc (1/2)') + assert.include(response.text(), 'one@test') + }) + + test('stats', async ({ client }) => { + const response = await client.get('/api/v1/stats') + response.assertStatus(200) + response.assertBodyContains({ sources: [] }) + }) + + test('completeness follows files and par2 lists through triggers', async ({ assert }) => { + const now = new Date() + const complete = async (id: number) => (await db.from('releases').where('id', id).first()).complete + const file = (releaseId: number, name: string) => ({ + filename: name, poster: 'p', date: now.getTime(), parts: 1, size: 100_000, release_id: releaseId, + groups: '[]', message_ids: '{}', created_at: now, updated_at: now, + }) + + // Counter rule: [i/M] announced 2 files. + const [counted] = await db.table('releases').insert({ release_key: 'c'.repeat(40), name: 'Counted', groups: '[]', file_total: 2, first_seen_at: now, last_seen_at: now }).returning('id') + await db.table('files').insert(file(counted.id, 'a.rar')) + assert.isFalse(await complete(counted.id)) + await db.table('files').insert(file(counted.id, 'b.rar')) + assert.isTrue(await complete(counted.id)) + + // par2 rule: the listed file becomes present once a file of its exact size is there. + const [listed] = await db.table('releases').insert({ release_key: 'l'.repeat(40), name: 'Listed', groups: '[]', first_seen_at: now, last_seen_at: now }).returning('id') + await db.table('release_files').insert({ release_id: listed.id, par2_file_id: '1'.repeat(32), name: 'Listed.mkv', size: 5_000_000, md5_16k: '0'.repeat(32) }) + assert.isFalse(await complete(listed.id)) + const [f] = await db.table('files').insert(file(listed.id, 'Xq8r2KdLm3Zw91PqA7')).returning('id') + const [b] = await db.table('binaries').insert({ file_key: 'z'.repeat(40), poster: 'p', subject: '"x" yEnc', name: 'x', total_parts: 1, groups: '[]', first_seen_at: now, last_seen_at: now, file_id: f.id }).returning('id') + assert.isFalse(await complete(listed.id)) + await db.from('binaries').where('id', b.id).update({ size: 5_000_000 }) + assert.isTrue(await complete(listed.id)) + }) + + test('par2-only releases are hidden unless asked for', async ({ client, assert }) => { + const now = new Date() + const [release] = await db.table('releases') + .insert({ release_key: 'p'.repeat(40), name: 'Webtest.Par2only.S01E01.1080p.WEB-DL-GRP', groups: '[]', first_seen_at: now, last_seen_at: now }) + .returning('id') + await db.table('release_files').insert({ release_id: release.id, par2_file_id: '2'.repeat(32), name: 'Webtest.Par2only.S01E01.mkv', size: 1_000_000_000, md5_16k: '0'.repeat(32) }) + await db.table('files').insert({ filename: 'Xq8r2KdLm3Zw91PqA7', content_type: 'par2', poster: 'p', date: now.getTime(), parts: 1, size: 3919, + release_id: release.id, groups: '[]', message_ids: '{}', created_at: now, updated_at: now }) + + const hidden = await client.get('/api/v1/releases').qs({ q: 'webtest par2only' }) + assert.notInclude(hidden.body().releases.map((r: any) => r.id), Number(release.id)) + const shown = await client.get('/api/v1/releases').qs({ q: 'webtest par2only', par2_only: '1' }) + const row = shown.body().releases.find((r: any) => r.id === Number(release.id)) + assert.isTrue(row.par2Only) + }) +}) diff --git a/tests/unit/content.spec.ts b/tests/unit/content.spec.ts new file mode 100644 index 0000000..4437a02 --- /dev/null +++ b/tests/unit/content.spec.ts @@ -0,0 +1,45 @@ +import { test } from '@japa/runner' +import yencode from 'yencode' +import { sniffContentType } from '#lib/content/index' +import { crc32Hex, decodeSegment, YencError } from '#lib/yenc/index' + +test.group('content / sniffContentType', () => { + test('recognises containers and media by magic bytes', ({ assert }) => { + const magic = (hex: string, pad = 16) => Buffer.concat([Buffer.from(hex, 'hex'), Buffer.alloc(pad)]) + assert.equal(sniffContentType(Buffer.concat([Buffer.from('nzbe'), Buffer.alloc(40)])), 'nzbe') + assert.equal(sniffContentType(magic('5041523200504b54')), 'par2') + assert.equal(sniffContentType(magic('526172211a070100')), 'rar5') + assert.equal(sniffContentType(magic('526172211a0700')), 'rar4') + assert.equal(sniffContentType(magic('1a45dfa3')), 'matroska') + assert.equal(sniffContentType(magic('0000001866747970')), 'mp4') + assert.equal(sniffContentType(magic('28b52ffd')), 'zstd') + assert.equal(sniffContentType(Buffer.from('\n')), 'nzb') + assert.isNull(sniffContentType(Buffer.from('plain text'))) + assert.equal(sniffContentType(magic('ffd8ffe0')), 'jpeg') + assert.equal(sniffContentType(magic('89504e470d0a1a0a')), 'png') + assert.equal(sniffContentType(Buffer.concat([Buffer.from('ID3'), Buffer.alloc(16)])), 'mp3') + assert.equal(sniffContentType(Buffer.concat([Buffer.from('fLaC'), Buffer.alloc(16)])), 'flac') + assert.equal(sniffContentType(Buffer.from('RIFF\x00\x00\x00\x00AVI LIST', 'latin1')), 'avi') + + // UDF (Blu-ray) volume recognition sequence at 32 KiB + const udf = Buffer.alloc(0x9000) + udf.write('BEA01', 0x8001, 'latin1') + udf.write('NSR03', 0x8801, 'latin1') + assert.equal(sniffContentType(udf), 'udf') + }) +}) + +test.group('yenc / lenientSize', () => { + test('decodes a part whose declared size leaves out a header, still checking the CRC', ({ assert }) => { + const data = Buffer.concat([Buffer.from('nzbe'), Buffer.alloc(16, 7), Buffer.from('ciphertext-here')]) + const declared = data.length - 20 + const body = Buffer.from( + `=ybegin part=1 line=128 size=${declared} name=f6dd6b156692a52b.bin\r\n=ypart begin=1 end=${declared}\r\n` + + yencode.encode(data).toString('latin1') + + `\r\n=yend size=${declared} part=1 pcrc32=${crc32Hex(data)}`, + 'latin1' + ) + assert.throws(() => decodeSegment(body), YencError as any) + assert.isTrue(decodeSegment(body, { lenientSize: true }).data.equals(data)) + }) +}) diff --git a/tests/unit/nntp_group.spec.ts b/tests/unit/nntp_group.spec.ts new file mode 100644 index 0000000..0fa682c --- /dev/null +++ b/tests/unit/nntp_group.spec.ts @@ -0,0 +1,18 @@ +import { test } from '@japa/runner' +import { parseGroupResponse } from '#lib/nntp/group' + +test.group('nntp / parseGroupResponse', () => { + test('parses the reply from news.eweka.nl', ({ assert }) => { + // Captured 2026-09-27. count == last - first + 1, so the field order is standard. + assert.deepEqual(parseGroupResponse('211 27275164317 266599 27275430915 alt.binaries.test'), { + count: 27275164317n, + first: 266599n, + last: 27275430915n, + name: 'alt.binaries.test', + }) + }) + + test('rejects non-211 replies', ({ assert }) => { + assert.throws(() => parseGroupResponse('411 No such group'), /Unexpected GROUP response/) + }) +}) diff --git a/tests/unit/nzb.spec.ts b/tests/unit/nzb.spec.ts new file mode 100644 index 0000000..d36640b --- /dev/null +++ b/tests/unit/nzb.spec.ts @@ -0,0 +1,55 @@ +import { test } from '@japa/runner' +import { buildNzb } from '#lib/nzb/index' + +test.group('nzb / buildNzb', () => { + test('writes files, groups and ordered segments', ({ assert }) => { + const xml = buildNzb( + [ + { + name: 'Some.Show.S01E01.mkv', + poster: 'poster@example.invalid', + date: 1_790_000_000_000, + groups: ['alt.binaries.test'], + segments: { '2': { id: 'b@x', size: 700 }, '1': { id: 'a@x', size: 800 } }, + }, + ], + { title: 'Some.Show.S01E01' } + ) + assert.include(xml, 'Some.Show.S01E01') + assert.include(xml, 'subject=""Some.Show.S01E01.mkv" yEnc (1/2)"') + assert.include(xml, 'date="1790000000"') + assert.include(xml, 'alt.binaries.test') + assert.isBelow(xml.indexOf('>a@x<'), xml.indexOf('>b@x<')) + assert.include(xml, 'a@x') + }) +}) + +test.group('nzb / parseNzb', () => { + test('reads meta, files, groups and segments, decoding entities', async ({ assert }) => { + const { parseNzb } = await import('#lib/nzb/index') + const xml = ` + + + Some.Show.S01E01s3cr&t + + alt.binaries.bloaf + + b@x.local + <a@x.local> + + + +` + const nzb = parseNzb(xml)! + assert.deepEqual(nzb.meta, { name: 'Some.Show.S01E01', password: 's3cr&t' }) + assert.lengthOf(nzb.files, 1) + assert.equal(nzb.files[0].subject, 'Wond40 "Nachbe" x.part1.rar" yEnc (0001/2)') + assert.deepEqual(nzb.files[0].groups, ['alt.binaries.bloaf']) + assert.deepEqual(nzb.files[0].segments, [ + { number: 1, bytes: 2000, id: 'a@x.local' }, + { number: 2, bytes: 1000, id: 'b@x.local' }, + ]) + assert.isNull(parseNzb('not an nzb')) + }) +}) diff --git a/tests/unit/par2.spec.ts b/tests/unit/par2.spec.ts new file mode 100644 index 0000000..d6e2119 --- /dev/null +++ b/tests/unit/par2.spec.ts @@ -0,0 +1,60 @@ +import { test } from '@japa/runner' +import { createHash } from 'node:crypto' +import { isPar2, parsePar2 } from '#lib/par2/index' + +const md5 = (data: Buffer) => createHash('md5').update(data).digest() + +/** Builds one PAR2 packet with a valid checksum. */ +function packet(setId: Buffer, type: string, body: Buffer) { + const padded = Buffer.concat([body, Buffer.alloc((4 - (body.length % 4)) % 4)]) + const length = Buffer.alloc(8) + length.writeBigUInt64LE(BigInt(64 + padded.length)) + const typeBuf = Buffer.alloc(16) + typeBuf.write(type, 'latin1') + const hashed = Buffer.concat([setId, typeBuf, padded]) + return Buffer.concat([Buffer.from('PAR2\0PKT', 'latin1'), length, md5(hashed), hashed]) +} + +function fileDesc(setId: Buffer, name: string, size: number) { + const len = Buffer.alloc(8) + len.writeBigUInt64LE(BigInt(size)) + const md5_16k = md5(Buffer.from(`${name}-16k`)) + const fileId = md5(Buffer.concat([md5_16k, len, Buffer.from(name)])) + const body = Buffer.concat([fileId, md5(Buffer.from(name)), md5_16k, len, Buffer.from(name, 'utf8')]) + return packet(setId, 'PAR 2.0\0FileDesc', body) +} + +test.group('par2 / parsePar2', () => { + const setId = Buffer.alloc(16, 7) + + test('reads file names and sizes from FileDesc packets', ({ assert }) => { + const data = Buffer.concat([ + packet(setId, 'PAR 2.0\0Main', Buffer.alloc(12)), + fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600), + fileDesc(setId, 'Some.Movie.2024.part02.rar', 52428800), + fileDesc(setId, 'Some.Movie.2024.part01.rar', 104857600), // duplicates are common + ]) + assert.isTrue(isPar2(data)) + const set = parsePar2(data)! + assert.equal(set.recoverySetId, setId.toString('hex')) + assert.deepEqual( + set.files.map((f) => [f.name, f.length]), + [ + ['Some.Movie.2024.part01.rar', 104857600], + ['Some.Movie.2024.part02.rar', 52428800], + ] + ) + }) + + test('skips corrupted packets and resumes at the next one', ({ assert }) => { + const bad = fileDesc(setId, 'bad.bin', 1) + bad[100] ^= 0xff + const data = Buffer.concat([bad, fileDesc(setId, 'good.bin', 12345)]) + assert.deepEqual(parsePar2(data)!.files.map((f) => f.name), ['good.bin']) + }) + + test('returns null for non-par2 data', ({ assert }) => { + assert.isFalse(isPar2(Buffer.from('Rar!\x1a\x07\x00', 'latin1'))) + assert.isNull(parsePar2(Buffer.from('hello world'))) + }) +}) diff --git a/tests/unit/rar.spec.ts b/tests/unit/rar.spec.ts new file mode 100644 index 0000000..442fc5d --- /dev/null +++ b/tests/unit/rar.spec.ts @@ -0,0 +1,230 @@ +import { test } from '@japa/runner' +import { crc32 } from 'node:zlib' +import { decodeRar4Name, parseRarHeaders, rarFormat, storedEntryData, type RarEntry } from '#lib/rar/index' + +/** Entries without the extraction fields, for comparing listings. */ +const listing = (entries: RarEntry[]) => entries.map(({ stored, dataOffset, ...rest }) => rest) + +// --- RAR 4 builders ----------------------------------------------------------- + +const RAR4 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x00]) + +function rar4Block(type: number, flags: number, body: Buffer) { + const header = Buffer.alloc(7) + header.writeUInt8(type, 2) + header.writeUInt16LE(flags, 3) + header.writeUInt16LE(7 + body.length, 5) + const block = Buffer.concat([header, body]) + block.writeUInt16LE(crc32(block.subarray(2)) & 0xffff, 0) + return block +} + +function rar4Main(flags: number) { + return rar4Block(0x73, flags, Buffer.alloc(6)) +} + +function rar4File(name: Buffer, opts: { flags?: number; packed: number; unpacked: number; method?: number }) { + const body = Buffer.alloc(25) + body.writeUInt8(opts.method ?? 0x33, 18) + body.writeUInt32LE(opts.packed, 0) + body.writeUInt32LE(opts.unpacked, 4) + body.writeUInt16LE(name.length, 19) + // LONG_BLOCK (0x8000) is always set on file headers. + return Buffer.concat([rar4Block(0x74, 0x8000 | (opts.flags ?? 0), Buffer.concat([body, name])), Buffer.alloc(opts.packed)]) +} + +// --- RAR 5 builders ----------------------------------------------------------- + +const RAR5 = Buffer.from([0x52, 0x61, 0x72, 0x21, 0x1a, 0x07, 0x01, 0x00]) + +function vint(value: number) { + const bytes: number[] = [] + do { + let byte = value % 128 + value = Math.floor(value / 128) + if (value > 0) byte |= 0x80 + bytes.push(byte) + } while (value > 0) + return Buffer.from(bytes) +} + +function rar5Block(type: number, headerFlags: number, fields: Buffer, opts: { extra?: Buffer; data?: number } = {}) { + let flags = headerFlags + const parts = [vint(type)] + const sizes: Buffer[] = [] + if (opts.extra) { + flags |= 0x0001 + sizes.push(vint(opts.extra.length)) + } + if (opts.data !== undefined) { + flags |= 0x0002 + sizes.push(vint(opts.data)) + } + const header = Buffer.concat([...parts, vint(flags), ...sizes, fields, opts.extra ?? Buffer.alloc(0)]) + const sized = Buffer.concat([vint(header.length), header]) + const crc = Buffer.alloc(4) + crc.writeUInt32LE(crc32(sized) >>> 0) + return Buffer.concat([crc, sized, Buffer.alloc(opts.data ?? 0)]) +} + +function rar5File(name: string, size: number, opts: { packed: number; headerFlags?: number; encrypted?: boolean; dir?: boolean }) { + const nameBuf = Buffer.from(name, 'utf8') + const fields = Buffer.concat([ + vint(opts.dir ? 0x0001 : 0), // file flags + vint(size), + vint(0x20), // attributes + vint(0), // compression info + vint(0), // host OS + vint(nameBuf.length), + nameBuf, + ]) + // Encryption record: size, type 1, then its data (dummy here). + const extra = opts.encrypted ? Buffer.concat([vint(3), vint(0x01), Buffer.from([0, 0])]) : undefined + return rar5Block(2, opts.headerFlags ?? 0, fields, { extra, data: opts.packed }) +} + +test.group('rar / RAR 4', () => { + test('lists files of a first volume, including 64-bit sizes and flags', ({ assert }) => { + const data = Buffer.concat([ + RAR4, + rar4Main(0x0001 | 0x0100), // volume, first volume + rar4File(Buffer.from('Some.Movie.2024/movie.mkv'), { packed: 10, unpacked: 5_000_000, flags: 0x0002 }), + ]) + assert.equal(rarFormat(data), 'rar4') + const info = parseRarHeaders(data)! + assert.include(info, { format: 'rar4', volume: true, firstVolume: true, headersEncrypted: false, truncated: true }) + assert.deepEqual(listing(info.entries), [ + { name: 'Some.Movie.2024/movie.mkv', size: 5_000_000, packedSize: 10, directory: false, encrypted: false, splitBefore: false, splitAfter: true }, + ]) + }) + + test('detects password-protected files and encrypted headers', ({ assert }) => { + const files = parseRarHeaders( + Buffer.concat([RAR4, rar4Main(0), rar4File(Buffer.from('secret.bin'), { packed: 4, unpacked: 4, flags: 0x0004 })]) + )! + assert.isTrue(files.entries[0].encrypted) + const headers = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0x0080)]))! + assert.isTrue(headers.headersEncrypted) + assert.lengthOf(headers.entries, 0) + }) + + test('copies out stored entries, not compressed ones', ({ assert }) => { + const data = Buffer.concat([ + RAR4, + rar4Main(0), + rar4File(Buffer.from('a.nzb'), { packed: 5, unpacked: 5, method: 0x30 }), + rar4File(Buffer.from('b.nzb'), { packed: 5, unpacked: 9 }), + ]) + const [stored, packed] = parseRarHeaders(data)!.entries + data.write('', stored.dataOffset) + assert.equal(storedEntryData(data, stored)?.toString(), '') + assert.isFalse(packed.stored) + assert.isNull(storedEntryData(data, packed)) + }) + + test('stops at a header with a bad CRC', ({ assert }) => { + const good = rar4File(Buffer.from('a.bin'), { packed: 1, unpacked: 1 }) + const bad = rar4File(Buffer.from('b.bin'), { packed: 1, unpacked: 1 }) + bad[10] ^= 0xff + const info = parseRarHeaders(Buffer.concat([RAR4, rar4Main(0), good, bad]))! + assert.deepEqual(info.entries.map((e) => e.name), ['a.bin']) + }) + + test('decodes compressed Unicode names', ({ assert }) => { + // "é.txt": ASCII fallback "?.txt", then highByte 0, flags 0b01_00_00_00 (é via high byte, + // then 3 plain bytes), followed by one more flag byte for the last byte. + const ascii = Buffer.from('?.txt', 'latin1') + const encoded = Buffer.from([0x00, 0b01000000, 0xe9, 0x2e, 0x74, 0x78, 0b00000000, 0x74]) + assert.equal(decodeRar4Name(Buffer.concat([ascii, Buffer.from([0]), encoded])), 'é.txt') + assert.equal(decodeRar4Name(Buffer.from('plain.txt')), 'plain.txt') + }) +}) + +test.group('rar / RAR 5', () => { + test('lists files and reads volume flags', ({ assert }) => { + const data = Buffer.concat([ + RAR5, + rar5Block(1, 0, vint(0x0001)), // main: volume, no volume number → first volume + rar5File('Release/track01.flac', 30_000_000, { packed: 20, headerFlags: 0x0010 }), + ]) + assert.equal(rarFormat(data), 'rar5') + const info = parseRarHeaders(data)! + assert.include(info, { format: 'rar5', volume: true, firstVolume: true, headersEncrypted: false }) + assert.deepEqual(listing(info.entries), [ + { name: 'Release/track01.flac', size: 30_000_000, packedSize: 20, directory: false, encrypted: false, splitBefore: false, splitAfter: true }, + ]) + }) + + test('a later volume carries a volume number', ({ assert }) => { + const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, Buffer.concat([vint(0x0003), vint(4)]))]))! + assert.isFalse(info.firstVolume) + }) + + test('detects encrypted files and encrypted headers', ({ assert }) => { + const files = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('x.mkv', 1, { packed: 1, encrypted: true })]))! + assert.isTrue(files.entries[0].encrypted) + const headers = parseRarHeaders(Buffer.concat([RAR5, rar5Block(4, 0, Buffer.alloc(3))]))! + assert.isTrue(headers.headersEncrypted) + }) + + test('marks the listing complete at the end-of-archive header', ({ assert }) => { + const info = parseRarHeaders(Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a', 1, { packed: 1 }), rar5Block(5, 0, vint(0))]))! + assert.isFalse(info.truncated) + assert.lengthOf(info.entries, 1) + }) + + test('copies out entries stored without compression', ({ assert }) => { + const data = Buffer.concat([RAR5, rar5Block(1, 0, vint(0)), rar5File('a.nzb', 5, { packed: 5 }), rar5Block(5, 0, vint(0))]) + const entry = parseRarHeaders(data)!.entries[0] + assert.isTrue(entry.stored) + data.write('', entry.dataOffset) + assert.equal(storedEntryData(data, entry)?.toString(), '') + assert.isNull(storedEntryData(data.subarray(0, entry.dataOffset + 2), entry)) + assert.isNull(storedEntryData(data, { ...entry, encrypted: true })) + }) + + test('returns null for other data', ({ assert }) => { + assert.isNull(parseRarHeaders(Buffer.from('PAR2\0PKT', 'latin1'))) + }) +}) + +test.group('rar / release naming from contents', () => { + const info = (names: string[]) => + ({ format: 'rar5', volume: false, firstVolume: true, solid: false, headersEncrypted: false, truncated: true, + entries: names.map((name) => ({ name, size: 1, packedSize: 1, directory: false, encrypted: false, splitBefore: false, splitAfter: false })) }) as any + + test('uses the shared top-level directory when it looks like a release name', async ({ assert }) => { + const { releaseNameFromContents } = await import('#services/ArchiveService') + assert.equal( + releaseNameFromContents(info(['GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM/00-a.jpg', 'GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM/01-b.flac'])), + 'GLDY_LX_-_Pretty_Girl_Rage_EP-WEB-2026-DIM' + ) + // No directory: the largest readable entry names the release. + assert.equal( + releaseNameFromContents(info(['Joker.Folie.a.Deux.2024.2160p.HMAX.WEB-DL.MULTi.DDP5.1.Atmos.H.265.nzb'])), + 'Joker.Folie.a.Deux.2024.2160p.HMAX.WEB-DL.MULTi.DDP5.1.Atmos.H.265' + ) + assert.isUndefined(releaseNameFromContents(info(['movie.mkv']))) + assert.isUndefined(releaseNameFromContents(info(['Xq8r2KdLm3Zw91PqA7.mkv']))) + assert.isUndefined(releaseNameFromContents(info(['a/x', 'b/y']))) + assert.isUndefined(releaseNameFromContents(info(['Xq8r2KdLm3Zw91Pq/x.mkv']))) + }) + + test('recognises obfuscated names', async ({ assert }) => { + const { looksRandom } = await import('#services/ArchiveService') + assert.isTrue(looksRandom('CGBWgdhlSCAOAW62gLWRRM')) + assert.isTrue(looksRandom('0220e975b1417f96b77dc64410e10ebc')) + assert.isFalse(looksRandom('Plus.and.Minus.S01E06.1080p')) + assert.isFalse(looksRandom('1790546808_nicovideo_jp_watch_sm26679715')) + }) +}) + +test.group('rar / looksRandom capitals', () => { + test('long letter-only tokens with many capitals are random', async ({ assert }) => { + const { looksRandom } = await import('#services/ArchiveService') + assert.isTrue(looksRandom('RmISakytFwXSCEzjVEpaKk')) + assert.isTrue(looksRandom('326678d1d07ef12a.bin')) + assert.isFalse(looksRandom('Some.Show.S01E01')) + assert.isFalse(looksRandom('TheQuickBrownFoxJumps')) + }) +}) diff --git a/tests/unit/release.spec.ts b/tests/unit/release.spec.ts new file mode 100644 index 0000000..e65f038 --- /dev/null +++ b/tests/unit/release.spec.ts @@ -0,0 +1,169 @@ +import { test } from '@japa/runner' +import { fileStem, releaseIdentity, subjectRemainder } from '#lib/release/index' +import { looksLikeFilename, weakFileKey } from '#lib/subject/index' + +test.group('release / fileStem', () => { + test('reduces all files of a set to the same stem', ({ assert }) => { + const names = [ + '1790546808_nicovideo_jp_watch_sm26679715.tar.zst', + '1790546808_nicovideo_jp_watch_sm26679715.tar.zst.par2', + '1790546808_nicovideo_jp_watch_sm26679715.tar.zst.vol03+04.par2', + ] + assert.deepEqual(new Set(names.map(fileStem)), new Set(['1790546808_nicovideo_jp_watch_sm26679715'])) + }) + + test('handles rar, split and par2 volume naming', ({ assert }) => { + assert.equal(fileStem('lJzuoIKA3T8xjzAD3vqtw0.part036.rar'), 'lJzuoIKA3T8xjzAD3vqtw0') + assert.equal(fileStem('Some.Movie.2024.r07'), 'Some.Movie.2024') + assert.equal(fileStem('Some.Movie.2024.mkv.001'), 'Some.Movie.2024') + assert.equal(fileStem('Some.Movie.2024.2160p.WEB-DL.H.265.mkv'), 'Some.Movie.2024.2160p.WEB-DL.H.265') + assert.equal(fileStem('Some.Movie.2024.H.264.nzb'), 'Some.Movie.2024.H.264') + assert.equal(fileStem('Some.Movie.2024.vol127-128.par2'), 'Some.Movie.2024') + assert.equal(fileStem('Some.Movie.2024.nfo'), 'Some.Movie.2024') + assert.equal(fileStem('00-artist-album-web-2026.m3u'), '00-artist-album-web-2026') + assert.equal(fileStem('f6dd6b156692a52b.bin'), 'f6dd6b156692a52b') + }) + + test('never reduces a name to nothing', ({ assert }) => { + assert.equal(fileStem('.par2'), '.par2') + }) +}) + +test.group('release / subjectRemainder', () => { + test('is empty when the subject only has counters, filename and size', ({ assert }) => { + assert.equal(subjectRemainder('[1/4] - "x.tar.zst" yEnc 15411409'), '') + assert.equal(subjectRemainder('. - [00/88] - "60SHkTQSvn5RZGExwju2.nzb" yEnc'), '') + }) + + test('keeps a release name written outside the quotes', ({ assert }) => { + assert.equal( + subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc'), + 'Some Show S01E02 1080p' + ) + }) +}) + +test.group('release / releaseIdentity', () => { + test('groups a par2 set with its archive by stem, per poster', ({ assert }) => { + const base = { poster: 'p', weak: false } + const a = releaseIdentity({ ...base, subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })! + const b = releaseIdentity({ ...base, subject: '[3/4] - "x.tar.zst.vol01+02.par2" yEnc 2', name: 'x.tar.zst.vol01+02.par2' })! + assert.equal(a.key, b.key) + assert.equal(a.name, 'x') + assert.equal(a.fileTotal, 4) + const other = releaseIdentity({ ...base, poster: 'q', subject: '[1/4] - "x.tar.zst" yEnc 1', name: 'x.tar.zst' })! + assert.notEqual(a.key, other.key) + }) + + test('a par2 index posted without [i/M] counter joins its release', ({ assert }) => { + const base = { poster: 'nicovideo_archiver', weak: false } + const archive = releaseIdentity({ ...base, subject: '[1/6] - "x.tar.zst" yEnc 105054576', name: 'x.tar.zst' })! + const index = releaseIdentity({ ...base, subject: '"x.tar.zst.par2" yEnc 3100', name: 'x.tar.zst.par2' })! + assert.equal(index.key, archive.key) + assert.equal(archive.fileTotal, 6) + assert.isUndefined(index.fileTotal) + }) + + test('uses the subject text when it names the release', ({ assert }) => { + const base = { poster: 'p', weak: false } + const a = releaseIdentity({ ...base, subject: 'My.Release [01/40] - "abc.part01.rar" yEnc', name: 'abc.part01.rar' })! + const b = releaseIdentity({ ...base, subject: 'My.Release [40/40] - "abc.par2" yEnc', name: 'abc.par2' })! + assert.equal(a.key, b.key) + assert.equal(a.name, 'My Release') + }) + + test('obfuscated volumes group by stem regardless of the random poster', ({ assert }) => { + const a = releaseIdentity({ poster: '3Uzx7TTSFpLzz@ngPost.com', subject: '"lJzu.part033.rar" yEnc', name: 'lJzu.part033.rar', weak: true })! + const b = releaseIdentity({ poster: 'bGLd1oaXRdrK1@ngPost.com', subject: '"lJzu.part034.rar" yEnc', name: 'lJzu.part034.rar', weak: true })! + assert.equal(a.key, b.key) + }) + + test('random extensionless names cannot be grouped by name', ({ assert }) => { + const name = '204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231' + assert.isNull(releaseIdentity({ poster: 'x', subject: `"${name}" yEnc`, name, weak: true })) + }) +}) + +test.group('subject / weakFileKey', () => { + test('ignores random per-article names, keeps real filenames', ({ assert }) => { + assert.isFalse(looksLikeFilename('204b914a977e5f891590ff1099c7a8caf1a0be537bfa5bf7f3a8b1213231')) + assert.isTrue(looksLikeFilename('lJzuoIKA3T8xjzAD3vqtw0.part033.rar')) + // Same file, different random names per article: same key. + assert.equal(weakFileKey('aaaa1111', 5000, 7), weakFileKey('bbbb2222', 5000, 7)) + // Equal-sized volumes with real names: different keys. + assert.notEqual(weakFileKey('x.part01.rar', 5000, 7), weakFileKey('x.part02.rar', 5000, 7)) + assert.notEqual(weakFileKey('aaaa1111', 5000, 7), weakFileKey('aaaa1111', 5001, 7)) + }) +}) + +test.group('release / rarReleaseIdentity', () => { + test('groups volumes by the shared top-level directory', async ({ assert }) => { + const { rarReleaseIdentity } = await import('#lib/release/index') + const first = rarReleaseIdentity([ + { name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/00-cover.jpg', size: 589382 }, + { name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/01-track.flac', size: 30_000_000 }, + ])! + const later = rarReleaseIdentity([{ name: 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM/03-track.flac', size: 29_000_000 }])! + assert.equal(first.key, later.key) + assert.equal(first.name, 'LASZR_-_DYING_ALIVE-(PRO094)-WEB-2026-DIM') + assert.equal(first.source, 'rar') + }) + + test('without a directory, volumes of one file share its first entry', async ({ assert }) => { + const { rarReleaseIdentity } = await import('#lib/release/index') + const a = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])! + const b = rarReleaseIdentity([{ name: 'Some.Show.S01E01.1080p.mkv', size: 3_000_000_000 }])! + assert.equal(a.key, b.key) + assert.equal(a.name, 'Some.Show.S01E01.1080p') + assert.isNull(rarReleaseIdentity([])) + }) +}) + +test.group('release / namesRelease', () => { + test('random per-file tags and poster ids are not a release name', async ({ assert }) => { + const { namesRelease, subjectRemainder, releaseIdentity } = await import('#lib/release/index') + const subject = 'elys-de81bcabef5faf6a - [Nyh5Q07Y] "JkpNIFg7En50zU1U" yEnc' + assert.isFalse(namesRelease(subjectRemainder(subject))) + const a = releaseIdentity({ poster: 'elys', subject, name: 'JkpNIFg7En50zU1U', weak: false })! + assert.equal(a.source, 'stem') + assert.isTrue(namesRelease(subjectRemainder('Some.Show.S01E02.1080p [03/40] - "abc.part02.rar" yEnc'))) + assert.isTrue(namesRelease('My Release')) + }) +}) + +test.group('release / post id', () => { + test('groups random-named files of one post by poster, post id and file count', async ({ assert }) => { + const { releaseIdentity, isPostToken } = await import('#lib/release/index') + const base = { poster: 'jW344UtC8R2bZEegrVLz@x', weak: false } + const a = releaseIdentity({ ...base, subject: '83110629-n [10/18] - "eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k" yEnc', name: 'eiae6fulpolas2adfts6o8643n3e6j6ct8d3219aau0jc33ol9r50tohncgr46evpl0dqoqktfb7k' })! + const b = releaseIdentity({ ...base, subject: '83110629-n [01/18] - "2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0" yEnc', name: '2ef0vm4kb618nv17kn0qsb0j1ofh7pgqtk5a3v5lkqqf1f7g6ed0' })! + assert.equal(a.key, b.key) + assert.equal(a.source, 'post') + assert.equal(a.fileTotal, 18) + const other = releaseIdentity({ ...base, subject: '83110630-n [01/18] - "x" yEnc', name: 'x' })! + assert.notEqual(a.key, other.key) + + assert.isTrue(isPostToken('83110629 n')) + assert.isFalse(isPostToken('PRiVATE')) + assert.isFalse(isPostToken('')) + // Without a counter there is nothing to bound the post: fall back to the stem. + assert.equal(releaseIdentity({ ...base, subject: '83110629-n - "x.bin" yEnc', name: 'x.bin' })!.source, 'stem') + }) +}) + +test.group('release / par2 names', () => { + test('names a season pack after what its episodes share', async ({ assert }) => { + const { par2ReleaseName } = await import('#lib/release/index') + const eps = Array.from({ length: 20 }, (_, i) => + `SEAL.Team.S03E${String(i + 1).padStart(2, '0')}.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato.mkv`) + assert.equal(par2ReleaseName(eps), 'SEAL.Team.S03.1080p.AMZN.WEB-DL.AAC2.0.H.265.DUAL-nitrato') + // Movie plus sample and nfo: the movie's (longest) name. + assert.equal( + par2ReleaseName(['Some.Movie.2024.1080p.BluRay.x264-GRP.mkv', 'Some.Movie.2024.1080p.BluRay.x264-GRP.nfo', 'sample.mkv']), + 'Some.Movie.2024.1080p.BluRay.x264-GRP' + ) + // Numbered parts: the shared name without the part number. + assert.equal(par2ReleaseName(['Album.Name-2024-GRP.part01.rar', 'Album.Name-2024-GRP.part02.rar']), 'Album.Name-2024-GRP') + assert.equal(par2ReleaseName(['only.one.file.2024.mkv']), 'only.one.file.2024') + }) +}) diff --git a/tests/unit/split.spec.ts b/tests/unit/split.spec.ts new file mode 100644 index 0000000..5b3d917 --- /dev/null +++ b/tests/unit/split.spec.ts @@ -0,0 +1,162 @@ +import { test } from '@japa/runner' +import { splitInstances, splitVerified, verifyVolume, type SplitArticle } from '#lib/split/index' +import { crc32Hex } from '#lib/yenc/index' + +/** Deterministic pseudo-random numbers. */ +function rng(seed: number) { + let x = seed + return () => { + x = (x * 1103515245 + 12345) & 0x7fffffff + return x / 0x7fffffff + } +} + +interface PostOptions { + volumes: number + parts: number + /** Parallel upload connections; they finish slightly out of order. */ + connections?: number + /** First article number of the post. */ + start?: number + /** Unrelated articles between two articles of this post (other posters). */ + noise?: number + /** Fraction of articles that never arrive. */ + drop?: number + label?: string + seed?: number +} + +/** Simulates a posting tool uploading equal-sized volumes. */ +function post(opts: PostOptions): SplitArticle[] { + const random = rng(opts.seed ?? 1) + const connections = opts.connections ?? 1 + const label = opts.label ?? 'p' + const queue: { volume: number; number: number }[] = [] + for (let v = 0; v < opts.volumes; v++) for (let n = 1; n <= opts.parts; n++) queue.push({ volume: v, number: n }) + + const articles: SplitArticle[] = [] + let articleNumber = opts.start ?? 1000 + for (let i = 0; i < queue.length; i += connections) { + // One round: each connection posts one article; they land in random order. + const round = queue.slice(i, i + connections).sort(() => random() - 0.5) + for (const { volume, number } of round) { + articleNumber += 1 + Math.floor(random() * (opts.noise ?? 0)) + if (random() < (opts.drop ?? 0)) continue + articles.push({ number, messageId: `${label}${volume}.${number}`, bytes: 700_000, articleNumber }) + } + } + return articles +} + +/** Source volume of an article, from its simulated message-id. */ +const source = (a: SplitArticle) => a.messageId.split('.')[0] + +/** Every result volume must contain articles of one source volume only. */ +function assertPure(assert: any, volumes: SplitArticle[][]) { + for (const volume of volumes) { + assert.lengthOf(new Set(volume.map(source)), 1, `mixed volume: ${volume.map((a) => a.messageId).join(' ')}`) + } +} + +test.group('split / splitInstances', () => { + test('separates sequentially posted volumes', ({ assert }) => { + const volumes = splitInstances(post({ volumes: 3, parts: 14 }), { totalParts: 14 }) + assert.lengthOf(volumes, 3) + assertPure(assert, volumes) + assert.isTrue(volumes.every((v) => v.length === 14)) + }) + + test('separates volumes posted over interleaved connections, out of order', ({ assert }) => { + for (const seed of [1, 2, 3, 4, 5]) { + const volumes = splitInstances(post({ volumes: 6, parts: 40, connections: 8, noise: 30, seed }), { totalParts: 40 }) + assertPure(assert, volumes) + assert.lengthOf(volumes, 6, `seed ${seed}`) + } + }) + + test('a volume that completes is never mixed, even with heavy loss', ({ assert }) => { + for (const seed of [11, 12, 13, 14, 15, 16, 17, 18]) { + const articles = post({ volumes: 5, parts: 30, connections: 6, noise: 20, drop: 0.2, seed }) + const volumes = splitInstances(articles, { totalParts: 30 }) + assertPure(assert, volumes.filter((v) => v.length === 30)) + } + }) + + test('a missing article only affects its own volume', ({ assert }) => { + const articles = post({ volumes: 4, parts: 20, connections: 4, drop: 0.05, seed: 7 }) + const volumes = splitInstances(articles, { totalParts: 20 }) + assertPure(assert, volumes) + assert.lengthOf(volumes, 4) + }) + + test('posts of the same size at different times do not mix', ({ assert }) => { + // An earlier post that lost articles (e.g. before the backfill window), then two more. + const articles = [ + ...post({ volumes: 3, parts: 14, connections: 4, drop: 0.3, label: 'a', start: 1_000, seed: 3 }), + ...post({ volumes: 3, parts: 14, connections: 4, label: 'b', start: 200_000, seed: 4 }), + ...post({ volumes: 2, parts: 14, connections: 2, label: 'c', start: 900_000, seed: 5 }), + ] + const volumes = splitInstances(articles, { totalParts: 14 }) + // The damaged post may produce mixed volumes, but those stay incomplete + // (they never become files); every volume that completes is pure. + const complete = volumes.filter((v) => v.length === 14) + assertPure(assert, complete) + assert.includeMembers(complete.map((v) => source(v[0])), ['b0', 'b1', 'b2', 'c0', 'c1']) + // The later, intact posts are not affected at all. + assertPure(assert, volumes.filter((v) => !source(v[0]).startsWith('a'))) + }) + + test('ignores duplicate message-ids and does not depend on input order', ({ assert }) => { + const articles = post({ volumes: 2, parts: 5, connections: 3, seed: 9 }) + const shuffled = [...articles].reverse().concat(articles[0]) + assert.deepEqual(splitInstances(shuffled, { totalParts: 5 }), splitInstances(articles, { totalParts: 5 })) + }) +}) + +/** Like post(), with real data per volume so part and file CRCs are genuine. */ +function postWithCrcs(opts: PostOptions): SplitArticle[] { + const random = rng((opts.seed ?? 1) + 1000) + const size = 64 + const data = new Map() + const articles = post(opts) + for (const a of articles) { + const volume = source(a) + if (!data.has(volume)) { + data.set(volume, Buffer.from(Array.from({ length: opts.parts * size }, () => Math.floor(random() * 256)))) + } + const file = data.get(volume)! + a.pcrc32 = crc32Hex(file.subarray((a.number - 1) * size, a.number * size)) + a.partSize = size + a.fileCrc32 = a.number === opts.parts ? crc32Hex(file) : null + } + return articles +} + +test.group('split / verified', () => { + test('verifyVolume accepts a correct volume and rejects a mixed one', ({ assert }) => { + const articles = postWithCrcs({ volumes: 2, parts: 6 }) + const volumeA = articles.filter((a) => source(a) === 'p0') + assert.isTrue(verifyVolume(volumeA, 6)) + const mixed = volumeA.map((a) => (a.number === 3 ? articles.find((b) => source(b) === 'p1' && b.number === 3)! : a)) + assert.isFalse(verifyVolume(mixed, 6)) + assert.isFalse(verifyVolume(volumeA.slice(0, 5), 6)) + }) + + test('only verified volumes come out, and they are never mixed', ({ assert }) => { + for (const seed of [21, 22, 23, 24, 25, 26]) { + const articles = [ + ...postWithCrcs({ volumes: 4, parts: 30, connections: 16, noise: 20, drop: 0.05, label: 'a', seed }), + ...postWithCrcs({ volumes: 3, parts: 30, connections: 16, noise: 20, label: 'b', start: 300_000, seed: seed + 50 }), + ] + for (const volume of splitVerified(articles, { totalParts: 30 })) { + assert.isTrue(verifyVolume(volume, 30)) + assertPure(assert, [volume]) + } + } + }) + + test('recovers every volume of a cleanly posted set', ({ assert }) => { + const volumes = splitVerified(postWithCrcs({ volumes: 5, parts: 20, connections: 4, noise: 10, seed: 31 }), { totalParts: 20 }) + assert.lengthOf(volumes, 5) + }) +}) diff --git a/tests/unit/spotnet.spec.ts b/tests/unit/spotnet.spec.ts new file mode 100644 index 0000000..d4fa202 --- /dev/null +++ b/tests/unit/spotnet.spec.ts @@ -0,0 +1,70 @@ +import { test } from '@japa/runner' +import { deflateRawSync } from 'node:zlib' +import { decodeSpotBinary, parseSpotArticleHeaders, parseSpotHeader, parseSpotXml, verifySpotHeader } from '#lib/spotnet/index' + +// A real spot from free.pt (self-signed, key 7), 2026-09-28. +const SPOT = {"from": "Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x.Fbc6Dm09v32bUgXpDT88QADBHELZ7wP4sMIMHKpQYxBIlK-pZveUFu0aMYUx-pUf6Z@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9-sgTnchacVAkC-p3USalOe-sacdamKdgXirL6>", "subject": "The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi | Bart", "id": "", "date": "Mon, 28 Sep 2026 18:24:45 GMT"} + +test.group('spotnet / headers', () => { + test('parses a spot header', ({ assert }) => { + const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)! + assert.include(spot, { + poster: 'Bart', + title: 'The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi', + tag: 'Bart', + category: 0, + keyId: 7, + size: 21651233204, + messageId: 'jWCbaAa4f1A7LC6agmOEx@spot.net', + }) + assert.deepEqual(spot.subcats, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] }) + }) + + test('verifies its signatures, and rejects another message id', ({ assert }) => { + const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)! + assert.isTrue(verifySpotHeader(spot)) + assert.isFalse(verifySpotHeader({ ...spot, messageId: 'other@spot.net' })) + assert.isNull(parseSpotHeader('someone ', 'subject', '')) + }) +}) + +test.group('spotnet / xml and nzb', () => { + test('reads the NZB segments and category from the X-XML lines', ({ assert }) => { + const raw = [ + 'From: Bart ', + 'X-XML: 7A & B0101a09', + 'X-XML: 01z00abc@spot.netdef@spot.net', + 'X-XML: 123', + ].join('\r\n') + const { headers, xml } = parseSpotArticleHeaders(raw) + assert.equal(headers.from, 'Bart ') + const spot = parseSpotXml(xml)! + assert.deepEqual(spot.nzbSegments, ['abc@spot.net', 'def@spot.net']) + assert.include(spot, { title: 'A & B', category: 0, size: 123 }) + assert.deepEqual(spot.subcats, ['01a09', '01z00']) + }) + + test('decodes an escaped, deflated NZB split over articles', ({ assert }) => { + const nzb = '\n\r\n=\0'.repeat(20) + const escaped = deflateRawSync(Buffer.from(nzb, 'latin1')) + .toString('latin1') + .replace(/[=\n\r\0]/g, (c) => ({ '=': '=D', '\n': '=C', '\r': '=B', '\0': '=A' })[c]!) + // Posted as lines, dot-stuffed, over two articles. + const lines = escaped.match(/.{1,40}/gs)!.map((l) => (l.startsWith('.') ? '.' + l : l)) + const half = Math.ceil(lines.length / 2) + const bodies = [lines.slice(0, half), lines.slice(half)].map((ls) => Buffer.from(ls.join('\r\n'), 'latin1')) + assert.equal(decodeSpotBinary(bodies).toString('latin1'), nzb) + }) +}) + +test.group('spotnet / moderation', () => { + test('recognises delete and dispose commands from key 2 only', async ({ assert }) => { + const { moderationTarget, spotterId } = await import('#lib/spotnet/index') + assert.equal(moderationTarget({ keyId: 2, title: 'DISPOSE 7IzfA0aCFTkOne6agvY06@spot.net - The Voice DE S15E07 (1080p)' }), '7IzfA0aCFTkOne6agvY06@spot.net') + assert.equal(moderationTarget({ keyId: 2, title: 'delete ' }), 'abc@spot.net') + assert.isNull(moderationTarget({ keyId: 7, title: 'DISPOSE abc@spot.net' })) + assert.isNull(moderationTarget({ keyId: 2, title: 'The Disposal 2024' })) + // Spotweb: base64 of the little-endian CRC32 of the key, without / + =. + assert.match(spotterId('0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE/9x'), /^[A-Za-z0-9]{6}$/) + }) +}) diff --git a/tests/unit/subject.spec.ts b/tests/unit/subject.spec.ts new file mode 100644 index 0000000..170e717 --- /dev/null +++ b/tests/unit/subject.spec.ts @@ -0,0 +1,108 @@ +import { test } from '@japa/runner' +import { fileKey, parseSubject } from '#lib/subject/index' +import { parseXrefGroups } from '#lib/nntp/xref' +import { inferTotalParts } from '#lib/yenc/index' + +test.group('subject / parseSubject', () => { + test('parses file counter, filename and segment counter', ({ assert }) => { + assert.deepEqual(parseSubject('. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc (56/274)'), { + name: 'obrMkrDeLmbhaouwYO34.part08.rar', + segment: 56, + totalSegments: 274, + fileIndex: 9, + fileTotal: 51, + base: '. - [09/51] - "obrMkrDeLmbhaouwYO34.part08.rar" yEnc', + }) + }) + + test('handles a file size after the segment counter', ({ assert }) => { + const subject = '[8/9] - "x.tar.zst.vol063+064.par2" yEnc (37/72) 51315352' + const parsed = parseSubject(subject)! + assert.equal(parsed.segment, 37) + assert.equal(parsed.totalSegments, 72) + assert.equal(parsed.base, '[8/9] - "x.tar.zst.vol063+064.par2" yEnc 51315352') + }) + + test('ignores counters inside the filename', ({ assert }) => { + const parsed = parseSubject('"Show (1/2) Special.mkv" yEnc (3/10)')! + assert.equal(parsed.name, 'Show (1/2) Special.mkv') + assert.equal(parsed.segment, 3) + assert.equal(parsed.totalSegments, 10) + }) + + test('the base is equal for all segments of one file', ({ assert }) => { + const a = parseSubject('[1/9] - "a.tar.zst" yEnc (253/1000) 716554110')! + const b = parseSubject('[1/9] - "a.tar.zst" yEnc (254/1000) 716554110')! + assert.equal(a.base, b.base) + }) + + test('returns null for obfuscated or invalid subjects', ({ assert }) => { + assert.isNull(parseSubject('GZFDJc0hB3Y0FwwAznkk')) + assert.isNull(parseSubject('"name.rar" yEnc')) + assert.isNull(parseSubject('"name.rar" yEnc (0/5)')) + assert.isNull(parseSubject('"name.rar" yEnc (6/5)')) + assert.isNull(parseSubject('"" yEnc (1/5)')) + }) +}) + +test.group('subject / fileKey', () => { + test('differs per poster and per segment count, is stable otherwise', ({ assert }) => { + const key = fileKey('poster@a', '"x.rar" yEnc', 10) + assert.lengthOf(key, 40) + assert.equal(key, fileKey('poster@a', '"x.rar" yEnc', 10)) + assert.notEqual(key, fileKey('poster@b', '"x.rar" yEnc', 10)) + assert.notEqual(key, fileKey('poster@a', '"x.rar" yEnc', 11)) + }) +}) + +test.group('nntp / parseXrefGroups', () => { + test('extracts crossposted groups', ({ assert }) => { + assert.deepEqual( + parseXrefGroups('news.eweka.nl alt.binaries.nl:35801424104 alt.binaries.test:27275430913'), + ['alt.binaries.nl', 'alt.binaries.test'] + ) + assert.deepEqual(parseXrefGroups(undefined), []) + }) +}) + +test.group('yenc / inferTotalParts', () => { + const size = 104857600 + const partSize = 384000 + + test('uses total= when present', ({ assert }) => { + assert.equal(inferTotalParts({ begin: { part: 2, total: 7, line: 128, size, name: 'a' } }), 7) + }) + + test('derives the total from the part size for middle and last parts', ({ assert }) => { + const expected = Math.ceil(size / partSize) // 274, as in the subject "(56/274)" + assert.equal(expected, 274) + assert.equal( + inferTotalParts({ + begin: { part: 56, line: 128, size, name: 'a' }, + part: { begin: 21120001, end: 21504000 }, + }), + expected + ) + assert.equal( + inferTotalParts({ + begin: { part: 274, line: 128, size, name: 'a' }, + part: { begin: 273 * partSize + 1, end: size }, + }), + 274 + ) + }) + + test('single-part posts have one part', ({ assert }) => { + assert.equal(inferTotalParts({ begin: { line: 128, size: 10, name: 'a' } }), 1) + }) +}) + +test.group('indexer / skipBodyMessageIds', () => { + test('matches the multi-group uploader, not other posts', async ({ assert }) => { + const { default: indexerConfig } = await import('#config/indexer') + const skip = (id: string) => indexerConfig.skipBodyMessageIds.some((re) => re.test(id)) + assert.isTrue(skip('')) + assert.isFalse(skip('')) + assert.isFalse(skip('')) + }) +}) diff --git a/tests/unit/yenc.spec.ts b/tests/unit/yenc.spec.ts new file mode 100644 index 0000000..190a807 --- /dev/null +++ b/tests/unit/yenc.spec.ts @@ -0,0 +1,192 @@ +import { test } from '@japa/runner' +import yencode from 'yencode' +import { + YencCrcError, + YencError, + YencFile, + crc32Hex, + decodeSegment, + parseYencMeta, +} from '#lib/yenc/index' + +/** Builds an article body as a yEnc poster would send it. */ +function article( + data: Buffer, + opts: { name: string; fileSize?: number; part?: number; total?: number; begin?: number; fileCrc?: string } +) { + const fileSize = opts.fileSize ?? data.length + const encoded = yencode.encode(data).toString('latin1') + const lines: string[] = [] + if (opts.part) { + lines.push(`=ybegin part=${opts.part} total=${opts.total} line=128 size=${fileSize} name=${opts.name}`) + const begin = opts.begin ?? 1 + lines.push(`=ypart begin=${begin} end=${begin + data.length - 1}`) + lines.push(encoded) + const fileCrc = opts.fileCrc ? ` crc32=${opts.fileCrc}` : '' + lines.push(`=yend size=${data.length} part=${opts.part} pcrc32=${crc32Hex(data)}${fileCrc}`) + } else { + lines.push(`=ybegin line=128 size=${fileSize} name=${opts.name}`) + lines.push(encoded) + lines.push(`=yend size=${data.length} crc32=${crc32Hex(data)}`) + } + return Buffer.from(lines.join('\r\n'), 'latin1') +} + +function randomBytes(length: number, seed = 1) { + const buf = Buffer.alloc(length) + let x = seed + for (let i = 0; i < length; i++) { + x = (x * 1103515245 + 12345) & 0x7fffffff + buf[i] = x & 0xff + } + return buf +} + +test.group('yenc / parseYencMeta', () => { + test('parses a multi-part header as seen on alt.binaries.test', ({ assert }) => { + const body = Buffer.from( + '=ybegin part=56 line=128 size=104857600 name=obrMkrDeLmbhaouwYO34.part08.rar\r\n' + + '=ypart begin=21120001 end=21504000\r\n' + + 'xyz\r\n' + + '=yend size=384000 part=56 pcrc32=B6339AFC', + 'latin1' + ) + const meta = parseYencMeta(body) + assert.deepEqual(meta.begin, { + part: 56, + total: undefined, + line: 128, + size: 104857600, + name: 'obrMkrDeLmbhaouwYO34.part08.rar', + }) + assert.deepEqual(meta.part, { begin: 21120001, end: 21504000 }) + assert.deepEqual(meta.end, { size: 384000, part: 56, pcrc32: 'b6339afc', crc32: undefined }) + assert.equal(body.subarray(meta.dataStart, meta.dataEnd).toString(), 'xyz') + }) + + test('keeps spaces and "=" in the filename', ({ assert }) => { + const body = article(Buffer.from('hi'), { name: 'My File = v2 (final).nfo' }) + assert.equal(parseYencMeta(body).begin.name, 'My File = v2 (final).nfo') + }) + + test('ignores leading blank lines before =ybegin', ({ assert }) => { + const body = Buffer.concat([Buffer.from('\r\n\r\n'), article(Buffer.from('hi'), { name: 'a.txt' })]) + assert.equal(parseYencMeta(body).begin.name, 'a.txt') + }) + + test('rejects bodies without yEnc markers', ({ assert }) => { + assert.throws(() => parseYencMeta(Buffer.from('plain text\r\n')), /=ybegin/) + assert.throws(() => parseYencMeta(Buffer.from('=ybegin line=128 size=2 name=a\r\nxx')), /=yend/) + }) +}) + +test.group('yenc / decodeSegment', () => { + test('round-trips a single-part post and checks crc32', ({ assert }) => { + const data = randomBytes(5000) + const segment = decodeSegment(article(data, { name: 'single.bin' })) + assert.isTrue(segment.data.equals(data)) + assert.equal(segment.offset, 0) + }) + + test('throws YencCrcError on a corrupted part', ({ assert }) => { + const body = article(randomBytes(1000), { name: 'x.bin', part: 1, total: 1 }) + const index = body.indexOf('\r\n=ypart') + 60 + body[index] = body[index] === 0x41 ? 0x42 : 0x41 + assert.throws(() => decodeSegment(body), YencCrcError as any) + }) + + test('undoes NNTP dot-stuffing', ({ assert }) => { + // Byte 4 encodes to "." (4 + 42 = 46). An unescaped leading "." is dot-stuffed to "..". + const data = Buffer.from([4, 5, 6]) + const body = Buffer.from( + `=ybegin line=128 size=3 name=dot.bin\r\n../0\r\n=yend size=3 crc32=${crc32Hex(data)}`, + 'latin1' + ) + assert.isTrue(decodeSegment(body).data.equals(data)) + assert.throws(() => decodeSegment(body, { stripDots: false }), YencError as any) + }) +}) + +test.group('yenc / YencFile', () => { + test('assembles parts added out of order and verifies the file crc', ({ assert }) => { + const file = randomBytes(10_000, 7) + const chunk = 3000 + const total = Math.ceil(file.length / chunk) + const bodies = Array.from({ length: total }, (_, i) => + article(file.subarray(i * chunk, (i + 1) * chunk), { + name: 'multi.rar', + fileSize: file.length, + part: i + 1, + total, + begin: i * chunk + 1, + fileCrc: i === total - 1 ? crc32Hex(file) : undefined, + }) + ) + + const segments = bodies.map((body) => decodeSegment(body)).reverse() + const yencFile = YencFile.fromSegment(segments[0]) + assert.equal(yencFile.totalParts, 4) + + for (const segment of segments.slice(0, -1)) yencFile.add(segment) + assert.isFalse(yencFile.isComplete) + assert.deepEqual(yencFile.missingParts(), [1]) + assert.throws(() => yencFile.getBuffer(), /missing parts: 1/) + + yencFile.add(segments.at(-1)!) + yencFile.add(segments[0]) // duplicate is ignored + assert.isTrue(yencFile.isComplete) + assert.isTrue(yencFile.getBuffer().equals(file)) + }) + + test('rejects segments of a file with another size', ({ assert }) => { + const a = decodeSegment(article(Buffer.from('aa'), { name: 'a.bin' })) + const b = decodeSegment(article(Buffer.from('bbb'), { name: 'b.bin' })) + assert.throws(() => YencFile.fromSegment(a).add(b), /does not belong to "a.bin"/) + }) + + test('accepts segments with per-article random names', ({ assert }) => { + const file = randomBytes(4000, 3) + const [x, y] = [0, 1].map((i) => + decodeSegment( + article(file.subarray(i * 2000, (i + 1) * 2000), { + name: i === 0 ? 'Xq8r2Kd' : 'p0Lm3Zw', + fileSize: 4000, + part: i + 1, + total: 2, + begin: i * 2000 + 1, + }) + ) + ) + const assembled = YencFile.fromSegment(x) + assembled.add(x) + assembled.add(y) + assert.isTrue(assembled.getBuffer().equals(file)) + }) +}) + +test.group('content / firstChunkMd5', () => { + test('hashes the first 16 KiB of the decoded first part', async ({ assert }) => { + const { firstChunkMd5 } = await import('#services/ContentService') + const { createHash } = await import('node:crypto') + const file = randomBytes(40_000, 5) + const body = article(file.subarray(0, 20_000), { name: 'x.bin', fileSize: 40_000, part: 1, total: 2 }) + assert.equal(firstChunkMd5(body, 40_000), createHash('md5').update(file.subarray(0, 16384)).digest('hex')) + // A first part shorter than 16 KiB of a larger file cannot give the fingerprint. + const short = article(file.subarray(0, 1000), { name: 'x.bin', fileSize: 40_000, part: 1, total: 40 }) + assert.isUndefined(firstChunkMd5(short, 40_000)) + }) +}) + +test.group('yenc / YencFile without total=', () => { + test('infers the part count when =ybegin has no total', ({ assert }) => { + const file = randomBytes(5000, 11) + const strip = (body: Buffer) => Buffer.from(body.toString('latin1').replace(/ total=\d+/, ''), 'latin1') + const parts = [0, 1].map((i) => + decodeSegment(strip(article(file.subarray(i * 3000, (i + 1) * 3000), { name: 'n.nzb', fileSize: 5000, part: i + 1, total: 2, begin: i * 3000 + 1 }))) + ) + const assembled = YencFile.fromSegment(parts[0]) + assert.equal(assembled.totalParts, 2) + parts.forEach((p) => assembled.add(p)) + assert.isTrue(assembled.getBuffer().equals(file)) + }) +}) diff --git a/types/simple-yenc.d.ts b/types/simple-yenc.d.ts deleted file mode 100644 index 61789ba..0000000 --- a/types/simple-yenc.d.ts +++ /dev/null @@ -1 +0,0 @@ -declare module 'simple-yenc'; diff --git a/types/yencode.d.ts b/types/yencode.d.ts new file mode 100644 index 0000000..aff38db --- /dev/null +++ b/types/yencode.d.ts @@ -0,0 +1,15 @@ +declare module 'yencode' { + export function decode(data: Buffer, stripDots?: boolean): Buffer + export function encode(data: Buffer, lineSize?: number, column?: number): Buffer + /** Returns the CRC32 as a 4-byte big-endian buffer. */ + export function crc32(data: Buffer, initial?: Buffer): Buffer + /** CRC32 of A+B from crc(A), crc(B) and the length of B. */ + export function crc32_combine(crc1: Buffer, crc2: Buffer, len2: number): Buffer + const yencode: { + decode: typeof decode + encode: typeof encode + crc32: typeof crc32 + crc32_combine: typeof crc32_combine + } + export default yencode +}