diff --git a/.adonisjs/client/manifest.d.ts b/.adonisjs/client/manifest.d.ts index 852b3d3..9fc3356 100644 --- a/.adonisjs/client/manifest.d.ts +++ b/.adonisjs/client/manifest.d.ts @@ -5,10 +5,5 @@ /// /// -/// /// -/// /// -/// -/// -/// diff --git a/.adonisjs/server/controllers.ts b/.adonisjs/server/controllers.ts index 34f4bc9..202eb41 100644 --- a/.adonisjs/server/controllers.ts +++ b/.adonisjs/server/controllers.ts @@ -6,8 +6,5 @@ export const controllers = { AccessTokens: () => import('#controllers/access_tokens_controller'), NewAccount: () => import('#controllers/new_account_controller'), - Newznab: () => import('#controllers/newznab_controller'), Profile: () => import('#controllers/profile_controller'), - Releases: () => import('#controllers/releases_controller'), - Seed: () => import('#controllers/seed_controller'), } diff --git a/.adonisjs/server/routes.d.ts b/.adonisjs/server/routes.d.ts index 37d0ca6..9aa155f 100644 --- a/.adonisjs/server/routes.d.ts +++ b/.adonisjs/server/routes.d.ts @@ -4,27 +4,15 @@ type ParamValue = string | number | bigint | boolean export type ScannedRoutes = { ALL: { - 'releases.index': { paramsTuple?: []; params?: {} } - 'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } - 'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } - 'releases.stats': { paramsTuple?: []; params?: {} } 'auth.new_account.store': { paramsTuple?: []; params?: {} } 'auth.access_tokens.store': { paramsTuple?: []; params?: {} } 'profile.profile.show': { paramsTuple?: []; params?: {} } 'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} } } GET: { - 'releases.index': { paramsTuple?: []; params?: {} } - 'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } - 'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } - 'releases.stats': { paramsTuple?: []; params?: {} } 'profile.profile.show': { paramsTuple?: []; params?: {} } } HEAD: { - 'releases.index': { paramsTuple?: []; params?: {} } - 'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } - 'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} } - 'releases.stats': { paramsTuple?: []; params?: {} } 'profile.profile.show': { paramsTuple?: []; params?: {} } } POST: { diff --git a/.adonisjs/server/routes.json b/.adonisjs/server/routes.json deleted file mode 100644 index b0765b6..0000000 --- a/.adonisjs/server/routes.json +++ /dev/null @@ -1 +0,0 @@ -{"root":[{"domain":"root","pattern":"/api/v1/releases","matchers":{},"tokens":[{"old":"/api/v1/releases","type":0,"val":"api","end":""},{"old":"/api/v1/releases","type":0,"val":"v1","end":""},{"old":"/api/v1/releases","type":0,"val":"releases","end":""}],"meta":{"params":[]},"name":"releases.index","handler":{"method":"index","reference":[null,"index"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id","type":1,"val":"id","end":"","matcher":{}}],"meta":{"params":["id"]},"name":"releases.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id/nzb","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id/nzb","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id/nzb","type":1,"val":"id","end":"","matcher":{}},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"nzb","end":""}],"meta":{"params":["id"]},"name":"releases.nzb","handler":{"method":"nzb","reference":[null,"nzb"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/stats","matchers":{},"tokens":[{"old":"/api/v1/stats","type":0,"val":"api","end":""},{"old":"/api/v1/stats","type":0,"val":"v1","end":""},{"old":"/api/v1/stats","type":0,"val":"stats","end":""}],"meta":{"params":[]},"name":"releases.stats","handler":{"method":"stats","reference":[null,"stats"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/signup","matchers":{},"tokens":[{"old":"/api/v1/auth/signup","type":0,"val":"api","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"signup","end":""}],"meta":{"params":[]},"name":"auth.new_account.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/new_account_controller')","name":"NewAccount"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/login","matchers":{},"tokens":[{"old":"/api/v1/auth/login","type":0,"val":"api","end":""},{"old":"/api/v1/auth/login","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/login","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/login","type":0,"val":"login","end":""}],"meta":{"params":[]},"name":"auth.access_tokens.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/profile","matchers":{},"tokens":[{"old":"/api/v1/account/profile","type":0,"val":"api","end":""},{"old":"/api/v1/account/profile","type":0,"val":"v1","end":""},{"old":"/api/v1/account/profile","type":0,"val":"account","end":""},{"old":"/api/v1/account/profile","type":0,"val":"profile","end":""}],"meta":{"params":[]},"name":"profile.profile.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/profile_controller')","name":"Profile"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/logout","matchers":{},"tokens":[{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],"meta":{"params":[]},"name":"profile.access_tokens.destroy","handler":{"method":"destroy","reference":[null,"destroy"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}}]} \ No newline at end of file diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 864de83..0000000 --- a/.dockerignore +++ /dev/null @@ -1,11 +0,0 @@ -node_modules -build -tmp -debug -.git -.idea -.vscode -.env -.env.* -!.env.example -*.log diff --git a/.env.example b/.env.example index f153372..ee3ebe3 100644 --- a/.env.example +++ b/.env.example @@ -12,9 +12,14 @@ APP_URL=http://${HOST}:${PORT} # Session SESSION_DRIVER=cookie +# Database (the rebuild's own) +DB_HOST=127.0.0.1 +DB_PORT=5432 +DB_USER=usenet_indexer +DB_PASSWORD= +DB_DATABASE=usenet_rebuild + #-------------------------------------------------------------------- # CORS (configure allowed origins for API access) #-------------------------------------------------------------------- -# CORS_ORIGIN=http://localhost:5173,http://localhost:3000 -# Newznab API key (/api?apikey=...); leave empty to allow access without a key -NEWZNAB_API_KEY= +# CORS_ORIGIN=http://localhost:5173,http://localhost:3000 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 3b1dab0..0000000 --- a/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# syntax=docker/dockerfile:1 - -# Node is pinned: on 24.21, jsonschema 1.5.0 (used by Ace to validate commands) -# throws "Invalid URL" on $ref, so no ace command starts. See LOG.md. -ARG NODE_VERSION=24.16 - -# Build stage: dev dependencies, TypeScript compile, native addons. -# The full node image ships python3/make/g++, needed to compile yencode. -FROM node:${NODE_VERSION}-bookworm AS build -WORKDIR /app -COPY package.json package-lock.json ./ -RUN npm ci -COPY . . -RUN node ace build \ - && cd build \ - && npm ci --omit=dev - -# Runtime stage: compiled app with production dependencies only. -FROM node:${NODE_VERSION}-bookworm-slim -ENV NODE_ENV=production -# bsdtar (libarchive) unpacks compressed RAR entries, e.g. NZBs posted inside RARs. -RUN apt-get update \ - && apt-get install -y --no-install-recommends libarchive-tools \ - && rm -rf /var/lib/apt/lists/* -WORKDIR /app -COPY --from=build /app/build ./ -USER node -# Every service picks its own command, e.g. `node ace worker:fetch`. -CMD ["node", "ace", "list"] diff --git a/adonisrc.ts b/adonisrc.ts index 832e211..71a369f 100644 --- a/adonisrc.ts +++ b/adonisrc.ts @@ -1,4 +1,4 @@ -import { indexEntities } from '@adonisjs/core' +import { indexEntities } from '@adonisjs/core/generators' import { defineConfig } from '@adonisjs/core/app' import { generateRegistry } from '@tuyau/core/hooks' @@ -53,7 +53,6 @@ export default defineConfig({ () => import('@adonisjs/cors/cors_provider'), () => import('@adonisjs/auth/auth_provider'), () => import('#providers/api_provider'), - () => import('@adonisjs/static/static_provider') ], /* @@ -104,10 +103,7 @@ export default defineConfig({ | the production build. | */ - metaFiles: [{ - pattern: 'public/**', - reloadServer: false, - }], + metaFiles: [], hooks: { init: [ diff --git a/app/controllers/newznab_controller.ts b/app/controllers/newznab_controller.ts deleted file mode 100644 index 7ee24d6..0000000 --- a/app/controllers/newznab_controller.ts +++ /dev/null @@ -1,123 +0,0 @@ -import type { HttpContext } from '@adonisjs/core/http' -import env from '#start/env' -import SearchService from '#services/SearchService' -import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index' - -const LIMITS = { max: 100, default: 50 } - -/** Search functions and the category each defaults to when `cat` is absent. */ -const SEARCH_FUNCTIONS: Record = { - 'search': undefined, - 'tvsearch': 5000, - 'tv-search': 5000, - 'movie': 2000, - 'movie-search': 2000, - 'music': 3000, - 'audio': 3000, - 'book': 7000, -} - -/** The configured key, if any (a Secret from .env, or a plain string set at runtime). */ -function configuredApiKey(): string | undefined { - const value: any = env.get('NEWZNAB_API_KEY') - const key = typeof value === 'string' ? value : value?.release() - return key || undefined -} - -function int(value: unknown): number | undefined { - return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : undefined -} - -/** - * Newznab API (CONCEPT.md D23) for Prowlarr, Sonarr, Radarr and the like: - * GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=… - */ -export default class NewznabController { - async handle({ request, response }: HttpContext) { - const qs = request.qs() - const t = String(qs.t ?? '') - const server = this.server(request) - - const xml = (body: string, type = 'application/xml') => { - response.header('Content-Type', `${type}; charset=utf-8`) - return response.send(body) - } - - const apiKey = configuredApiKey() - if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials')) - - if (t === 'caps') return xml(capsXml(server, LIMITS)) - - if (t === 'get') { - const id = int(qs.id) - if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)')) - const nzb = await SearchService.nzb(id) - if (!nzb) return xml(errorXml(300, 'No such item')) - const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb` - response.header('Content-Disposition', `attachment; filename="${filename}"`) - return xml(nzb.xml, 'application/x-nzb') - } - - if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`)) - - const requested = String(qs.cat ?? '') - .split(',') - .map((c) => int(c.trim())) - .filter((c): c is number => c !== undefined) - const fallback = SEARCH_FUNCTIONS[t] - const categories = expandCategories(requested.length ? requested : fallback ? [fallback] : []) - const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max) - const offset = int(qs.offset) ?? 0 - - // ID-only lookups (tvdbid, imdbid, …) are not supported: caps says so, but - // answer with an empty result rather than everything. - const q = typeof qs.q === 'string' ? qs.q.trim() : '' - const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p]) - - const result = idOnly - ? { total: 0, releases: [] } - : await SearchService.search({ - q, - categories, - readable: true, - notIncomplete: true, - hideContainers: true, - maxAgeDays: int(qs.maxage), - season: int(qs.season), - episode: int(qs.season) !== undefined ? int(qs.ep) : undefined, - sort: 'newest', - perPage: limit, - offset, - }) - - const items = result.releases.map((r: any) => { - const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name) - return { - id: r.id, - title: r.name, - category: r.category ?? 8010, - size: r.size, - files: r.fileCount, - postedAt: r.postedAt ? new Date(r.postedAt) : null, - poster: r.poster, - groups: r.groups ?? [], - // 1 only when a password is needed that we can't give; a known one is in our NZB. - password: r.passwordState === 'missing', - description: r.spotDescription ?? undefined, - season: tv ? `S${tv[1].padStart(2, '0')}` : undefined, - episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined, - } - }) - return xml(resultsXml(server, items, offset, result.total), 'application/rss+xml') - } - - private server(request: HttpContext['request']): NewznabServer { - const proto = request.header('x-forwarded-proto') ?? 'http' - const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost' - return { - title: 'Usenet Indexer', - baseUrl: `${proto}://${host}`, - apiKey: configuredApiKey(), - } - } -} diff --git a/app/controllers/releases_controller.ts b/app/controllers/releases_controller.ts deleted file mode 100644 index d2fdc97..0000000 --- a/app/controllers/releases_controller.ts +++ /dev/null @@ -1,65 +0,0 @@ -import type { HttpContext } from '@adonisjs/core/http' -import SearchService, { type SortOrder } from '#services/SearchService' - -const flag = (value: unknown) => value === '1' || value === 'true' || value === true - -/** A size in MB from the query string, as bytes. */ -function megabytes(value: unknown): number | undefined { - const mb = Number(value) - return value !== undefined && value !== '' && Number.isFinite(mb) && mb >= 0 ? Math.round(mb * 1024 * 1024) : undefined -} - -export default class ReleasesController { - /** GET /api/v1/releases?q=&readable=1&complete=1&hide_encrypted=1&min_size=&max_size=(MB)&ext=mkv,flac&source=&poster=&sort=&page=&per_page= */ - async index({ request }: HttpContext) { - const qs = request.qs() - return SearchService.search({ - q: typeof qs.q === 'string' ? qs.q : undefined, - readable: flag(qs.readable), - complete: flag(qs.complete), - hideEncrypted: flag(qs.hide_encrypted), - includePar2Only: flag(qs.par2_only), - source: typeof qs.source === 'string' && qs.source ? qs.source : undefined, - poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined, - minSize: megabytes(qs.min_size), - maxSize: megabytes(qs.max_size), - extensions: typeof qs.ext === 'string' && qs.ext ? qs.ext.split(/[\s,]+/) : undefined, - sort: qs.sort as SortOrder | undefined, - page: Number(qs.page) || 1, - perPage: Number(qs.per_page) || 50, - // The page shows "1,000+"; counting further made filter clicks take seconds. - countLimit: 1000, - }) - } - - /** GET /api/v1/releases/:id */ - async show({ params, response }: HttpContext) { - const release = await SearchService.release(Number(params.id)) - return release ?? response.notFound({ error: 'Release not found' }) - } - - /** GET /api/v1/releases/:id/nzb */ - async nzb({ params, response }: HttpContext) { - const nzb = await SearchService.nzb(Number(params.id)) - if (!nzb) return response.notFound({ error: 'Release not found' }) - const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb` - response.header('Content-Type', 'application/x-nzb') - response.header('Content-Disposition', `attachment; filename="${filename}"`) - return response.send(nzb.xml) - } - - /** GET /api/v1/releases/:id/image: the image of the release's spot, read from Usenet. */ - async image({ params, response }: HttpContext) { - const image = await SearchService.spotImage(Number(params.id)) - if (!image) return response.notFound({ error: 'No image' }) - response.header('Content-Type', image.type) - // A spot's image never changes. - response.header('Cache-Control', 'public, max-age=604800, immutable') - return response.send(image.data) - } - - /** GET /api/v1/stats */ - async stats() { - return SearchService.stats() - } -} diff --git a/app/controllers/seed_controller.ts b/app/controllers/seed_controller.ts deleted file mode 100644 index ee79f61..0000000 --- a/app/controllers/seed_controller.ts +++ /dev/null @@ -1,156 +0,0 @@ -import type { HttpContext } from '@adonisjs/core/http' -import SeedOrchestrator, { LeaseLost } from '#services/SeedOrchestrator' -import SeedFetchPool from '#services/SeedFetchPool' -import RemoteContentService, { ArticleRelayOff, JobLost } from '#services/RemoteContentService' -import type { SeedBatch } from '#lib/seed/wire' - -/** - * The seed API (CONCEPT.md D38). Nodes make outgoing HTTPS requests only: - * lease a range, post batches of what they found, then finish or release - * the range. Bodies may be gzip-compressed (Content-Encoding: gzip). - */ -export default class SeedController { - /** POST /api/v1/seed/lease → 200 { range } or 204 when there's nothing to do. */ - async lease({ seedNode, response }: HttpContext) { - const range = await SeedOrchestrator.lease(seedNode.id) - return range ? { range } : response.noContent() - } - - /** POST /api/v1/seed/ranges/:id/batches with a SeedBatch. */ - async batch({ seedNode, params, request, response }: HttpContext) { - const batch = request.body() as SeedBatch - if (!Number.isSafeInteger(batch?.next) || !Array.isArray(batch.files) || !Array.isArray(batch.fragments)) { - return response.badRequest({ error: 'Expected { next, headers, files[], fragments[] }' }) - } - return this.leased(response, async () => { - const stored = await SeedOrchestrator.batch(seedNode.id, Number(params.id), { - ...batch, - headers: Number(batch.headers) || 0, - obfuscated: Number(batch.obfuscated) || 0, - deferred: Number(batch.deferred) || 0, - }) - return { stored } - }) - } - - /** - * GET /api/v1/seed/ranges/:id/chunk?from=&to= → the overviews of those - * articles as a zlib/gzip stream (relay nodes fetch through the indexer). - */ - async chunk({ seedNode, params, request, response }: HttpContext) { - if (SeedFetchPool.size === 0) return response.serviceUnavailable({ error: 'Relay is off (SEED_NNTP_CONNECTIONS=0); fetch directly' }) - const from = Number(request.input('from')) - const to = Number(request.input('to')) - if (!Number.isSafeInteger(from) || !Number.isSafeInteger(to)) return response.badRequest({ error: 'Expected ?from=&to=' }) - return this.leased(response, async () => { - try { - const raw = await SeedOrchestrator.chunk(seedNode.id, Number(params.id), from, to) - response.header('content-type', 'application/octet-stream') - return response.send(raw) - } catch (error) { - if (error instanceof RangeError) return response.badRequest({ error: error.message }) - throw error - } - }) - } - - /** POST /api/v1/seed/ranges/:id/done */ - async done({ seedNode, params, response }: HttpContext) { - return this.leased(response, async () => { - await SeedOrchestrator.finish(seedNode.id, Number(params.id)) - return { ok: true } - }) - } - - /** POST /api/v1/seed/ranges/:id/release { error?, uncounted? } */ - async release({ seedNode, params, request, response }: HttpContext) { - const error = request.input('error') - const uncounted = request.input('uncounted') === true - return this.leased(response, async () => { - await SeedOrchestrator.release(seedNode.id, Number(params.id), typeof error === 'string' ? error : undefined, uncounted) - return { ok: true } - }) - } - - /** - * POST /api/v1/seed/inspect/lease { max } → 200 { jobs } or 204: content - * inspection and archive walks for the node (CONCEPT.md D41). - */ - async inspectLease({ seedNode, request, response }: HttpContext) { - const max = Math.min(10, Math.max(1, Number(request.input('max')) || 1)) - const jobs = await RemoteContentService.lease(seedNode.id, max) - return jobs.length > 0 ? { jobs } : response.noContent() - } - - /** - * POST /api/v1/seed/inspect/done { queue, jobId, token, result } where - * result is the probe as a JSON string (the body parser would turn the - * empty strings inside it into nulls). - */ - async inspectDone({ request, response }: HttpContext) { - const { queue, jobId, token, result } = request.only(['queue', 'jobId', 'token', 'result']) - if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string' || typeof result !== 'string') { - return response.badRequest({ error: 'Expected { queue, jobId, token, result: JSON string }' }) - } - return this.job(response, () => RemoteContentService.finish(queue, jobId, token, JSON.parse(result))) - } - - /** POST /api/v1/seed/inspect/failed { queue, jobId, token, error } */ - async inspectFailed({ request, response }: HttpContext) { - const { queue, jobId, token, error } = request.only(['queue', 'jobId', 'token', 'error']) - if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') { - return response.badRequest({ error: 'Expected { queue, jobId, token, error }' }) - } - return this.job(response, () => RemoteContentService.fail(queue, jobId, token, String(error ?? 'unknown error'))) - } - - /** POST /api/v1/seed/inspect/release { queue, jobId, token }: a stopping node gives a job back. */ - async inspectRelease({ request, response }: HttpContext) { - const { queue, jobId, token } = request.only(['queue', 'jobId', 'token']) - if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') { - return response.badRequest({ error: 'Expected { queue, jobId, token }' }) - } - return this.job(response, () => RemoteContentService.release(queue, jobId, token)) - } - - /** GET /api/v1/seed/articles?id= → the raw article body; 410 when the server doesn't have it. */ - async article({ request, response }: HttpContext) { - const id = request.input('id') - if (typeof id !== 'string' || id.length === 0 || id.length > 250) return response.badRequest({ error: 'Expected ?id=' }) - try { - const body = await RemoteContentService.article(id) - if (!body) return response.gone({ error: 'No such article' }) - response.header('content-type', 'application/octet-stream') - return response.send(body) - } catch (error: any) { - if (error instanceof ArticleRelayOff) return response.serviceUnavailable({ error: error.message }) - return response.badGateway({ error: `NNTP: ${error.message}` }) - } - } - - /** GET /api/v1/seed/status */ - async status() { - return SeedOrchestrator.status() - } - - /** 409 when the job's lock is gone (it was given to another node meanwhile). */ - private async job(response: HttpContext['response'], action: () => Promise) { - try { - await action() - return { ok: true } - } catch (error) { - if (error instanceof JobLost) return response.conflict({ error: error.message }) - throw error - } - } - - /** 409 when the range was given to another node meanwhile; the node should drop it. */ - private async leased(response: HttpContext['response'], action: () => Promise) { - try { - return await action() - } catch (error) { - if (error instanceof LeaseLost) return response.conflict({ error: error.message }) - throw error - } - } -} diff --git a/app/middleware/seed_node_middleware.ts b/app/middleware/seed_node_middleware.ts deleted file mode 100644 index b8865d0..0000000 --- a/app/middleware/seed_node_middleware.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { HttpContext } from '@adonisjs/core/http' -import type { NextFn } from '@adonisjs/core/types/http' -import SeedOrchestrator from '#services/SeedOrchestrator' - -declare module '@adonisjs/core/http' { - interface HttpContext { - /** The seeding node that made this request (CONCEPT.md D38). */ - seedNode: { id: number; name: string } - } -} - -/** - * Seed API requests carry a node's bearer token (`node ace seed:token - * `); anything else gets 401. The seed API is the only part of the app - * reachable from the internet (traefik routes only /api/v1/seed there). - */ -export default class SeedNodeMiddleware { - async handle(ctx: HttpContext, next: NextFn) { - const header = ctx.request.header('authorization') ?? '' - const token = header.startsWith('Bearer ') ? header.slice(7).trim() : '' - const node = token ? await SeedOrchestrator.nodeForToken(token) : null - if (!node) return ctx.response.unauthorized({ error: 'Unknown or disabled seed node token' }) - ctx.seedNode = node - return next() - } -} diff --git a/app/models/base_model.ts b/app/models/base_model.ts deleted file mode 100644 index 44cf9ca..0000000 --- a/app/models/base_model.ts +++ /dev/null @@ -1,4 +0,0 @@ -import { BaseModel as AdonisBaseModel } from '@adonisjs/lucid/orm' - -export default class BaseModel extends AdonisBaseModel { -} diff --git a/app/models/file.ts b/app/models/file.ts deleted file mode 100644 index a82f5ed..0000000 --- a/app/models/file.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { DateTime } from 'luxon' -import { column } from '@adonisjs/lucid/orm' -import BaseModel from '#models/base_model' -import { loadedSegments, storedSegments } from '#services/FileStore' -import type { Segment } from '#lib/nzb/index' - -export default class File extends BaseModel { - @column({ isPrimary: true }) - declare id: number - - @column({ columnName: 'file_key' }) - declare fileKey: string | null - - @column() - declare filename: string - - @column() - declare poster: string - - @column() - declare date: number - - @column() - declare parts: number - - /** Stored compressed (see FileStore); the model sees the list. */ - @column({ prepare: (value: Segment[]) => storedSegments(value), consume: (value: Buffer | null) => loadedSegments(value) }) - declare segments: Segment[] - - @column({ columnName: 'first_message_id' }) - declare firstMessageId: string | null - - @column() - declare groups: string[] - - @column.dateTime({ autoCreate: true }) - declare createdAt: DateTime - - @column.dateTime({ autoCreate: true, autoUpdate: true }) - declare updatedAt: DateTime -} diff --git a/app/models/group.ts b/app/models/group.ts deleted file mode 100644 index b9ecd05..0000000 --- a/app/models/group.ts +++ /dev/null @@ -1,32 +0,0 @@ -import { DateTime } from 'luxon' -import { column } from '@adonisjs/lucid/orm' -import BaseModel from '#models/base_model' - -export default class Group extends BaseModel { - @column({ isPrimary: true }) - declare id: number - - @column() - declare name: string - - @column() - declare active: boolean - - /** Every article up to here was fetched successfully (see FetchRangeService). */ - @column({ columnName: 'last_indexed_id' }) - declare lastIndexedId: bigint - - /** Download bodies of obfuscated articles to identify them (bandwidth-heavy). */ - @column({ columnName: 'fetch_bodies' }) - declare fetchBodies: boolean - - /** How far the scheduler has queued fetch jobs. */ - @column({ columnName: 'last_queued_id' }) - declare lastQueuedId: bigint - - @column.dateTime({ autoCreate: true }) - declare createdAt: DateTime - - @column.dateTime({ autoCreate: true, autoUpdate: true }) - declare updatedAt: DateTime -} diff --git a/app/services/ArchiveService.ts b/app/services/ArchiveService.ts deleted file mode 100644 index 683187d..0000000 --- a/app/services/ArchiveService.ts +++ /dev/null @@ -1,170 +0,0 @@ -import db from '@adonisjs/lucid/services/db' -import logger from '@adonisjs/core/services/logger' -import FileStore from '#services/FileStore' -import IngestService from '#services/IngestService' -import type { RarInfo } from '#lib/rar/index' -import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index' - -/** Stored per file and release; a listing of thousands of files is not useful. */ -const MAX_ENTRIES = 500 - -export interface ArchiveResult { - info: RarInfo - releaseId?: number - releaseRenamed?: string - grouped?: boolean -} - -/** - * Records what a RAR volume's headers say: the listing on the file, the - * release it belongs to (from the shared directory or continued entry, when - * the file has none yet), and for first volumes the contents and password - * flag on the release. See CONCEPT.md (D19, D21). - */ -class ArchiveService { - async record(file: any, binary: any, info: RarInfo): Promise { - const { next: _next, ...listing } = info - const stored = { ...listing, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) } - await FileStore.setDetails(Number(file.id), { archive: stored }) - - let releaseId = file.release_id ?? binary?.release_id ?? null - let grouped = false - const identity = rarReleaseIdentity(info.entries) - if (!releaseId && identity) { - const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [ - { - release: identity, - poster: binary?.poster ?? file.poster, - weakIdentity: true, - groups: binary?.groups ?? file.groups ?? [], - postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined, - }, - ]) - releaseId = ids.get(identity.key)! - grouped = true - if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId }) - await db.from('files').where('id', file.id).update({ release_id: releaseId }) - } - if (!releaseId) return { info } - - const release = await db.from('releases').where('id', releaseId).first() - const update: Record = {} - if (info.firstVolume !== false) { - update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted) - if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries) - } else if (info.headersEncrypted) { - update.encrypted = true - } - const better = releaseNameFromContents(info) - // A spot's title is kept (CONCEPT.md D29). - const renamed = - release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined - if (renamed) update.name = renamed - if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update) - - if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`) - return { info, releaseId, releaseRenamed: renamed, grouped } - } -} - -/** - * A release name from an archive listing: the top-level directory shared by - * all entries when it reads like a release name, otherwise the stem of the - * largest entry with a readable name (e.g. an archive holding just - * `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped. - */ -export function releaseNameFromContents(info: Pick): string | undefined { - // Hidden entries name nothing: one poster adds the same 22 GB ".1917-….mkv" decoy to every upload (LOG.md 2026-10-06 (2)). - const listed = info.entries.filter((e) => !isHidden(e.name)) - const entries = listed.filter((e) => !e.directory) - const tops = new Set(listed.map((e) => e.name.split(/[\\/]/)[0])) - if (tops.size === 1 && listed.some((e) => /[\\/]/.test(e.name))) { - const [top] = tops - if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top - } - const candidates = entries - .map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 })) - .filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem)) - .filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem)) - .sort((a, b) => b.size - a.size) - return candidates[0]?.stem -} - -/** An entry in a hidden directory or a hidden file itself (`.x/…`, `dir/.x`). */ -export function isHidden(name: string): boolean { - return name.split(/[\\/]/).some((part) => part.startsWith('.')) -} - -/** - * Releases named after a hidden archive entry before those were skipped: - * renamed from their stored listing (no downloads). Used by - * `release:rename --hidden`. - */ -export async function renameHiddenNamed(): Promise { - const releases = await db - .from('releases') - .where('name', 'like', '.%') - .whereNot('source', 'spot') - .whereNotNull('contents') - .select('id', 'name', 'contents') - let renamed = 0 - for (const release of releases) { - const entries: any[] = Array.isArray(release.contents) ? release.contents : [] - const fromHidden = entries.some((e) => isHidden(e.name) && fileStem(String(e.name).split(/[\\/]/).pop() ?? '') === release.name) - if (!fromHidden) continue - const better = releaseNameFromContents({ entries }) - if (!better || better === release.name) continue - await db.from('releases').where('id', release.id).update({ name: better }) - renamed++ - } - return renamed -} - -/** - * Renames random-named releases from their stored archive listing (no - * downloads). Used by `release:rename`. - */ -export async function renameFromStoredContents(): Promise { - const releases = await db - .from('releases') - .whereNotNull('contents') - .whereRaw("jsonb_array_length(contents) > 0") - .select('id', 'name', 'source', 'contents') - let renamed = 0 - for (const release of releases) { - if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue - const better = releaseNameFromContents({ entries: release.contents }) - if (!better || better === release.name) continue - await db.from('releases').where('id', release.id).update({ name: better }) - renamed++ - } - return renamed -} - -/** - * Obfuscated names: one long token of mixed letters and digits, no - * separators, and the other generated shapes seen on Usenet (LOG.md - * 2026-10-01 (3)). Kept in step with SQL's is_readable_name(). - */ -export function looksRandom(name: string): boolean { - // An unknown short extension (e.g. ".bin") says nothing about the name. - name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '') - // VIDEO_TS, Sample, CD1: real words, but no name. - if (isGenericName(name)) return true - if (/^[0-9a-f]{16,}$/i.test(name)) return true - // 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time) - if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true - // UUIDs - if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true - // 1650777167 (yEncBin), "83111763 n" (rclone uploads) - if (/^\d{6,}( n)?$/.test(name)) return true - // prvjr7hapd0i1ny14pz4, and rclone's long base32 names - if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true - // wHJ7s71sAGE: short, but capitals all over the token, not only at word starts - if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true - if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false - // Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk). - return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6 -} - -export default new ArchiveService() diff --git a/app/services/ArchiveWalkService.ts b/app/services/ArchiveWalkService.ts deleted file mode 100644 index 67c4d44..0000000 --- a/app/services/ArchiveWalkService.ts +++ /dev/null @@ -1,160 +0,0 @@ -import db from '@adonisjs/lucid/services/db' -import env from '#start/env' -import NntpService from '#services/NntpService' -import QueueService from '#services/QueueService' -import { segmentsOf } from '#services/FileStore' -import { looksRandom, releaseNameFromContents } from '#services/ArchiveService' -import { rarVolume } from '#lib/content/index' -import type { RarEntry, RarInfo } from '#lib/rar/index' -import { probeWalk, type WalkProbe, type WalkTask } from '#lib/content/probe' - -/** Stored per release, as for a first-volume listing (ArchiveService). */ -const MAX_ENTRIES = 500 - -/** Listed sizes below this share of the release's size: the first segment listed only part of it. */ -const COVERED = 0.7 - -/** Inspection queues a walk this long after the first volume, when the release has had time to complete. */ -const WALK_DELAY_MS = 30 * 60 * 1000 - -export interface WalkOutcome { - releaseId: number - /** Nothing downloaded: why. */ - skipped?: 'missing' | 'covered' | 'volumes' - entries?: number - before?: number - complete?: boolean - stopped?: string - segments?: number - renamed?: string -} - -/** - * Complete archive listings by walking RAR headers (CONCEPT.md D40). The - * first segment of a first volume lists the entries that start in it: for - * a set of many small files (an audiobook's MP3s, a music album) that is - * the first one or two. The walk reads one segment per further entry and - * volume, at low priority: its own queue, one job at a time per collection - * worker, and NNTP connections only when inspection isn't waiting for one. - */ -class ArchiveWalkService { - /** Segments one walk may download (~0.7 MB each). */ - readonly maxSegments = env.get('ARCHIVE_WALK_MAX_SEGMENTS', 400) - - /** A listing that leaves most of the release unaccounted for. */ - worthWalking(entries: number, listedBytes: number, releaseSize: number): boolean { - if (entries === 0 || entries >= MAX_ENTRIES) return false - return listedBytes < COVERED * releaseSize - } - - /** After inspection listed a first RAR volume whose listing ends inside the set. */ - async queueAfterInspection(releaseId: number | null, info: RarInfo) { - if (!releaseId || info.format === '7z' || info.firstVolume === false) return - if (!info.truncated || info.headersEncrypted || info.entries.length === 0) return - await QueueService.archiveWalkQueue.add('walk', { releaseId }, { jobId: `walk-${releaseId}`, delay: WALK_DELAY_MS }) - } - - /** - * The database half before a walk (D41): the release's volumes, or the - * outcome when there's nothing to walk. - */ - async prepare(releaseId: number): Promise<{ task: WalkTask } | { outcome: WalkOutcome }> { - const release = await db.from('releases').where('id', releaseId).select('id', 'size', 'contents').first() - if (!release) return { outcome: { releaseId, skipped: 'missing' } } - const before = Array.isArray(release.contents) ? release.contents : [] - const listed = before.reduce((sum: number, e: any) => sum + Number(e.size ?? 0), 0) - if (!this.worthWalking(before.length, listed, Number(release.size))) { - return { outcome: await this.done(releaseId, { releaseId, skipped: 'covered' }) } - } - const volumes = await this.volumes(releaseId) - if (!volumes) return { outcome: await this.done(releaseId, { releaseId, skipped: 'volumes' }) } - return { - task: { - kind: 'walk', - releaseId, - volumes: volumes.map((file) => segmentsOf(file).map(({ number, id }) => ({ number, id }))), - maxSegments: this.maxSegments, - }, - } - } - - /** The database half after a walk: the fuller listing, the password flag and a better name. */ - async apply(releaseId: number, walked: WalkProbe): Promise { - const release = await db.from('releases').where('id', releaseId).select('id', 'name', 'source', 'contents', 'encrypted').first() - if (!release) return { releaseId, skipped: 'missing' } - const before = Array.isArray(release.contents) ? release.contents : [] - const outcome: WalkOutcome = { - releaseId, - before: before.length, - entries: walked.entries.length, - complete: walked.complete, - stopped: walked.stopped, - segments: walked.segments, - } - if (walked.entries.length > before.length) { - const contents = walked.entries.slice(0, MAX_ENTRIES) - const update: Record = { - contents: JSON.stringify(contents), - encrypted: Boolean(release.encrypted) || walked.headersEncrypted || walked.entries.some((e) => e.encrypted), - } - const better = releaseNameFromContents({ entries: walked.entries as RarEntry[] }) - // The rule of ArchiveService.record: random and post-subject names give way, a spot's title stays. - if (better && better !== release.name && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post')) { - update.name = better - outcome.renamed = better - } - await db.from('releases').where('id', releaseId).update(update) - } - return this.done(releaseId, outcome) - } - - /** Walks a release here, on the inspection pool's connections when inspection isn't waiting for one. */ - async walk(releaseId: number): Promise { - const prepared = await this.prepare(releaseId) - if ('outcome' in prepared) return prepared.outcome - return this.apply(releaseId, await probeWalk(prepared.task, fetchLow)) - } - - private async done(releaseId: number, outcome: WalkOutcome) { - await db.from('releases').where('id', releaseId).update({ contents_walked_at: new Date() }) - return outcome - } - - /** - * The release's RAR volumes in order, from volume 1 up to the first gap. - * Null without a first volume (e.g. volumes with random names). - */ - private async volumes(releaseId: number): Promise { - const files = await db.from('files').where('release_id', releaseId).select('id', 'filename', 'real_name', 'segments') - const sets = new Map>() - for (const file of files) { - const volume = rarVolume(String(file.real_name ?? file.filename)) - if (!volume || segmentsOf(file).length === 0) continue - const set = sets.get(volume.base) ?? new Map() - if (!set.has(volume.number)) set.set(volume.number, file) - sets.set(volume.base, set) - } - // A release holds one set, now and then a second (a sample, subs): the biggest. - const set = [...sets.values()].sort((a, b) => b.size - a.size)[0] - if (!set?.has(1)) return null - const ordered: any[] = [] - for (let n = 1; set.has(n); n++) ordered.push(set.get(n)) - return ordered - } -} - -/** One article body on a pooled connection, behind inspection's waiting jobs. */ -async function fetchLow(messageId: string): Promise { - const conn = await NntpService.acquire(true) - let failed = false - try { - return await NntpService.timed(NntpService.fetchBody(conn, messageId), 'BODY') - } catch (error) { - failed = true - throw error - } finally { - NntpService.release(conn, failed) - } -} - -export default new ArchiveWalkService() diff --git a/app/services/BackfillService.ts b/app/services/BackfillService.ts deleted file mode 100644 index 600a3de..0000000 --- a/app/services/BackfillService.ts +++ /dev/null @@ -1,112 +0,0 @@ -import db from '@adonisjs/lucid/services/db' -import logger from '@adonisjs/core/services/logger' -import indexerConfig from '#config/indexer' -import NntpService from '#services/NntpService' -import FetchRangeService from '#services/FetchRangeService' -import { articleAtDate } from '#lib/nntp/date_search' - -/** Articles XOVERed to find a posting date near an article number. */ -const PROBE = 200n - -/** - * Header backfill (CONCEPT.md D32). `start` gives a group a cursor from - * where live indexing began back to the article posted N days ago; `tick` - * (every scheduler run) queues low-priority, header-only fetch ranges from - * that cursor downwards, newest first, while few are waiting and the - * database stays within its size limits. - */ -class BackfillService { - /** Sets the group's backfill to reach back `days` days. Returns its range. */ - async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> { - const group = await db.from('groups').where('name', groupName).first() - if (!group) throw new Error(`Unknown group ${groupName}`) - const conn = await NntpService.acquire() - let failed = false - try { - const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`) - const dateOf = async (n: bigint) => { - const { overviews } = await NntpService.timed(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`) - const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1] - const date = header ? Date.parse(header.date) : Number.NaN - return Number.isNaN(date) ? null : date - } - const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf) - // Live indexing began a little before the group was added: continue from there. - const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf) - const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began - await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() }) - return next > until ? { from: until, to: next } : null - } catch (error) { - failed = true - throw error - } finally { - NntpService.release(conn, failed) - } - } - - /** Stops a group's backfill (queued ranges still run). */ - async stop(groupName: string) { - await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null }) - } - - /** Queues the next backfill ranges. Returns how many. */ - async tick(): Promise { - const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill - const groups = await db - .from('groups') - .where('active', true) - .whereNotNull('backfill_until_id') - .whereRaw('backfill_next_id > backfill_until_id') - .orderBy('id') - if (groups.length === 0) return 0 - - const room = await this.room(maxDefaultGb, maxHddGb) - if (room !== true) { - logger.warn(`Backfill paused: ${room}.`) - return 0 - } - const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count') - let free = maxQueued - Number(count) - let queued = 0 - // Round-robin over the groups, one range each per pass. - while (free > 0 && groups.length > 0) { - for (const group of [...groups]) { - if (free <= 0) break - const next = BigInt(group.backfill_next_id) - const until = BigInt(group.backfill_until_id) - const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n - await FetchRangeService.queue(group, start, next, { backfill: true }) - group.backfill_next_id = (start - 1n).toString() - await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id }) - free-- - queued++ - if (start - 1n <= until) { - groups.splice(groups.indexOf(group), 1) - logger.info(`${group.name}: backfill fully queued.`) - } - } - } - return queued - } - - /** True when both parts of the database are under their limits, else why not. */ - /** - * Whether the database has room for more history: the default tablespace - * (NVMe) and tablespace `hdd` against BACKFILL_MAX_NVME_GB/_HDD_GB. Seed - * leases (CONCEPT.md D38) check the same limits. - */ - async room( - maxDefaultGb = indexerConfig.backfill.maxDefaultGb, - maxHddGb = indexerConfig.backfill.maxHddGb - ): Promise { - const { rows } = await db.rawQuery( - `SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')` - ) - const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3 - if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})` - if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})` - return true - } -} - -export default new BackfillService() diff --git a/app/services/BinaryFileService.ts b/app/services/BinaryFileService.ts deleted file mode 100644 index e7700d6..0000000 --- a/app/services/BinaryFileService.ts +++ /dev/null @@ -1,144 +0,0 @@ -import db from '@adonisjs/lucid/services/db' -import FileStore from '#services/FileStore' -import SfvService from '#services/SfvService' -import ContentService from '#services/ContentService' -import InspectionQueue from '#services/InspectionQueue' - -export interface BinaryFilesResult { - /** New files. */ - saved: number - /** Binaries whose file already existed (same first segment or file_key). */ - existing: number - /** Conflicted, gone or already linked: nothing to do. */ - skipped: number - /** Not all parts stored yet (the job is retried for these). */ - incomplete: number[] - inspections: number -} - -/** - * Turns completed binaries into files (worker:file), a batch at a time - * (LOG.md 2026-10-06 (7)): one query each for the binaries, their parts and - * files that exist already, one INSERT for the new files (so the files - * statement trigger updates each release once), and one UPDATE and DELETE - * for the binaries' links and copied parts. It was one job of ~8 queries per - * binary, ~700 a minute. - */ -class BinaryFileService { - async store(binaryIds: number[]): Promise { - const result: BinaryFilesResult = { saved: 0, existing: 0, skipped: 0, incomplete: [], inspections: 0 } - const ids = [...new Set(binaryIds)] - const binaries = await db - .from('binaries') - .whereIn('id', ids) - .whereNull('conflicted_at') - .whereNull('file_id') - .select('id', 'file_key', 'name', 'real_name', 'poster', 'posted_at', 'groups', 'release_id', 'total_parts') - result.skipped = ids.length - binaries.length - if (binaries.length === 0) return result - - const partsOf = new Map() - const parts = await db - .from('parts') - .whereIn('binary_id', binaries.map((b) => b.id)) - .orderBy([{ column: 'binary_id' }, { column: 'number' }]) - .select('binary_id', 'number', 'message_id', 'bytes') - for (const part of parts) { - const list = partsOf.get(Number(part.binary_id)) ?? [] - list.push({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) }) - partsOf.set(Number(part.binary_id), list) - } - const complete = binaries.filter((b) => { - const ok = (partsOf.get(Number(b.id))?.length ?? 0) >= b.total_parts - if (!ok) result.incomplete.push(Number(b.id)) - return ok - }) - if (complete.length === 0) return result - - // Already known as a file, e.g. added from a posted NZB before its own articles completed (id only: the row holds the segment list). - const firsts = complete.map((b) => partsOf.get(Number(b.id))![0].id) - const byFirst = new Map() - for (const row of await db.from('files').whereIn('first_message_id', firsts).select('id', 'first_message_id')) { - byFirst.set(row.first_message_id, Number(row.id)) - } - const fileIdOf = new Map() - const fresh = complete.filter((b) => { - const existing = byFirst.get(partsOf.get(Number(b.id))![0].id) - if (existing) fileIdOf.set(Number(b.id), existing) - return !existing - }) - - const now = new Date() - // file_key is unique: a second completion of the same upload inserts nothing. - const inserted = await FileStore.insert( - fresh.map((b) => { - const segments = partsOf.get(Number(b.id))! - return { - file_key: b.file_key, - filename: b.name, - poster: b.poster, - date: b.posted_at ? new Date(b.posted_at).getTime() : now.getTime(), - parts: segments.length, - segments, - groups: JSON.stringify(b.groups), - release_id: b.release_id, - real_name: b.real_name, - size: segments.reduce((sum, s) => sum + s.size, 0), - created_at: now, - updated_at: now, - } - }) - ) - const insertedByKey = new Map(inserted.map((row) => [row.file_key, row.id])) - const missing = fresh.filter((b) => !insertedByKey.has(b.file_key)).map((b) => b.file_key) - const byKey = new Map() - if (missing.length > 0) { - for (const row of await db.from('files').whereIn('file_key', missing).select('id', 'file_key')) byKey.set(row.file_key, Number(row.id)) - } - for (const b of fresh) { - const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key) - if (id) fileIdOf.set(Number(b.id), id) - } - const fresher = new Map(fresh.map((b) => [b.file_key, b])) - result.saved = inserted.length - result.existing = fileIdOf.size - inserted.length - - // Links, locked in file_key order like ingest's upsert, so the two can't deadlock. - const linked = complete.filter((b) => fileIdOf.has(Number(b.id))).sort((a, b) => a.file_key.localeCompare(b.file_key)) - if (linked.length > 0) { - await db.transaction(async (trx) => { - await trx.rawQuery( - `SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`, - [linked.map((b) => b.id)] - ) - // Files queued for inspection below keep their binary until it has its fingerprint (par2 naming, D16). - const queued = new Set(inserted.filter((row) => row.file_key && ContentService.isCandidate(String(fresher.get(row.file_key)?.name ?? ''))).map((row) => row.id)) - await trx.rawQuery( - `UPDATE binaries b SET file_id = v.file_id, awaiting_inspection = CASE WHEN v.queued THEN true ELSE b.awaiting_inspection END - FROM unnest(?::bigint[], ?::bigint[], ?::boolean[]) AS v(id, file_id, queued) WHERE b.id = v.id`, - [linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!), linked.map((b) => queued.has(fileIdOf.get(Number(b.id))!))] - ) - }) - // Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20). - await db.rawQuery(`DELETE FROM parts WHERE binary_id = ANY(?::bigint[]) AND pcrc32 IS NULL`, [linked.map((b) => b.id)]) - } - - // As for one file before: an SFV posted earlier may list them (D34), then inspection. - const added = inserted.flatMap((row) => { - const b = row.file_key ? fresher.get(row.file_key) : undefined - return b - ? [{ id: row.id, name: b.real_name ?? b.name, posted: String(b.name), poster: b.poster, date: b.posted_at ? new Date(b.posted_at).getTime() : null, release_id: b.release_id }] - : [] - }) - if (added.length > 0) await SfvService.claimAll(added.map(({ posted: _p, ...file }) => file)) - // By the posted name, as before: inspection skips later volumes and readable media names anyway. - const jobs = added - .filter((file) => ContentService.isCandidate(file.posted)) - .map((file) => file.id) - if (jobs.length > 0) await InspectionQueue.add(jobs) - result.inspections = jobs.length - return result - } -} - -export default new BinaryFileService() diff --git a/app/services/ContentService.ts b/app/services/ContentService.ts deleted file mode 100644 index 1b4bb5d..0000000 --- a/app/services/ContentService.ts +++ /dev/null @@ -1,321 +0,0 @@ -import db from '@adonisjs/lucid/services/db' -import logger from '@adonisjs/core/services/logger' -import NntpService from '#services/NntpService' -import Par2Service, { type Par2Result } from '#services/Par2Service' -import ArchiveService, { looksRandom, type ArchiveResult } from '#services/ArchiveService' -import NameService from '#services/NameService' -import { decodeSegment } from '#lib/yenc/index' -import NzbImportService, { type NzbImportResult } from '#services/NzbImportService' -import { - isArchiveStart, - isLaterVolume, - NICOVIDEO_FILE, - NICOVIDEO_RELEASE, - nicovideoReleaseName, - type ContentType, - type NicovideoInfo, -} from '#lib/content/index' -import { chunkMd5, probeInspection, type InspectProbe, type InspectTask, type TaskSegment } from '#lib/content/probe' -import type { Segment } from '#lib/nzb/index' -import { fileStem, releaseIdentity } from '#lib/release/index' -import IngestService from '#services/IngestService' -import { looksLikeFilename } from '#lib/subject/index' -import { sevenZipVolume } from '#lib/sevenzip/index' -import SevenZipService from '#services/SevenZipService' -import SfvService from '#services/SfvService' -import ArchiveWalkService from '#services/ArchiveWalkService' -import FileStore, { segmentsOf } from '#services/FileStore' -import GarbageFilter from '#services/GarbageFilter' - -/** - * MD5 of the first 16 KiB of a file, as par2 records it, from the body of - * the file's first part. Undefined when that part is shorter than 16 KiB but - * the file is not (the chunk would span two parts). - */ -export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined { - return chunkMd5(decodeSegment(body).data, fileSize) -} - -export interface ContentResult { - par2?: Par2Result - archive?: ArchiveResult - named: number - contentType: ContentType | null - nzb?: NzbImportResult - /** NZBs read from inside an archive. */ - archiveNzbs?: NzbImportResult[] - /** Files an SFV's listing moved into its release. */ - sfvMoved?: number - /** A niconico capture's release name from its API response (D34). */ - nicovideo?: string - /** The real name from the yEnc header, for a file posted under a random subject (D36). */ - yencName?: string - /** Why the file's release was deleted as garbage after this probe (D42). */ - garbage?: string -} - -/** - * Takes names and groupings from file contents (CONCEPT.md D21). For a - * completed file with a random name, one article (the first) is enough: - * - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry, - * - par2 files are recognised by content and name every file of their set, - * - RAR headers give the release grouping, name and contents. - * Inspection is prepare (database) → probe (downloads and parsing, - * lib/content/probe; here or on a seeding node, D41) → apply (database). - */ -class ContentService { - /** Random or missing names, and named first RAR volumes (for contents and password flag). */ - isCandidate(name: string) { - // Even under a random stem: the first volume and the index .par2 are inspected instead. - if (isLaterVolume(name)) return false - if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true - if (isArchiveStart(name)) return true - if (/\.nzb$/i.test(name)) return true - // SFVs name the files of their release; niconico captures carry their title (D34). - if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true - // Every 7z volume: the set is listed when its last volume is in (see inspect). - if (sevenZipVolume(name) !== null) return true - return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name)) - } - - /** - * Uploads that randomise the subject per article often keep the real file - * name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`; - * LOG.md 2026-10-02 (2)). A file with a random name takes that name, and - * leaves its weak one-file release for the release its name gives, where - * the other files of the upload gather. Returns the name, or undefined. - */ - private async nameFromYenc(file: any, binary: any, yencName: string): Promise { - const current = String(file.real_name ?? file.filename) - if (file.real_name || !looksRandom(fileStem(current))) return undefined - if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined - - const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null - const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1) - let releaseId = file.release_id - if (weak) { - const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true }) - if (identity) { - const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [ - { release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true }, - ]) - releaseId = ids.get(identity.key) ?? releaseId - } - } - await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId }) - if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId }) - return yencName - } - - /** - * Names a niconico capture's release after the video, while it still has - * the archiver's `