Rebuild: fresh AdonisJS 7 API app on Postgres
The rebuild branch starts over (REBUILD.md): the API starter kit, with Postgres instead of SQLite (its own database, DB_SEARCH_PATH for a shared one in development) and the #lib alias. Phase 1's code is imported from the current app module by module. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
699395c51f
commit
96f0ade10e
264 files changed
+777
-24132
No files matched your search
Vendored
-5
@@ -5,10 +5,5 @@
|
||||
|
||||
/// <reference path="../../adonisrc.ts" />
|
||||
/// <reference path="../../config/auth.ts" />
|
||||
/// <reference path="../../config/gc.ts" />
|
||||
/// <reference path="../../config/hash.ts" />
|
||||
/// <reference path="../../config/indexer.ts" />
|
||||
/// <reference path="../../config/logger.ts" />
|
||||
/// <reference path="../../config/nntp.ts" />
|
||||
/// <reference path="../../config/queue.ts" />
|
||||
/// <reference path="../../config/redis.ts" />
|
||||
@@ -6,8 +6,5 @@
|
||||
export const controllers = {
|
||||
AccessTokens: () => import('#controllers/access_tokens_controller'),
|
||||
NewAccount: () => import('#controllers/new_account_controller'),
|
||||
Newznab: () => import('#controllers/newznab_controller'),
|
||||
Profile: () => import('#controllers/profile_controller'),
|
||||
Releases: () => import('#controllers/releases_controller'),
|
||||
Seed: () => import('#controllers/seed_controller'),
|
||||
}
|
||||
Vendored
-12
@@ -4,27 +4,15 @@ type ParamValue = string | number | bigint | boolean
|
||||
|
||||
export type ScannedRoutes = {
|
||||
ALL: {
|
||||
'releases.index': { paramsTuple?: []; params?: {} }
|
||||
'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.stats': { paramsTuple?: []; params?: {} }
|
||||
'auth.new_account.store': { paramsTuple?: []; params?: {} }
|
||||
'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
|
||||
'profile.profile.show': { paramsTuple?: []; params?: {} }
|
||||
'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
|
||||
}
|
||||
GET: {
|
||||
'releases.index': { paramsTuple?: []; params?: {} }
|
||||
'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.stats': { paramsTuple?: []; params?: {} }
|
||||
'profile.profile.show': { paramsTuple?: []; params?: {} }
|
||||
}
|
||||
HEAD: {
|
||||
'releases.index': { paramsTuple?: []; params?: {} }
|
||||
'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.stats': { paramsTuple?: []; params?: {} }
|
||||
'profile.profile.show': { paramsTuple?: []; params?: {} }
|
||||
}
|
||||
POST: {
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
{"root":[{"domain":"root","pattern":"/api/v1/releases","matchers":{},"tokens":[{"old":"/api/v1/releases","type":0,"val":"api","end":""},{"old":"/api/v1/releases","type":0,"val":"v1","end":""},{"old":"/api/v1/releases","type":0,"val":"releases","end":""}],"meta":{"params":[]},"name":"releases.index","handler":{"method":"index","reference":[null,"index"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id","type":1,"val":"id","end":"","matcher":{}}],"meta":{"params":["id"]},"name":"releases.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id/nzb","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id/nzb","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id/nzb","type":1,"val":"id","end":"","matcher":{}},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"nzb","end":""}],"meta":{"params":["id"]},"name":"releases.nzb","handler":{"method":"nzb","reference":[null,"nzb"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/stats","matchers":{},"tokens":[{"old":"/api/v1/stats","type":0,"val":"api","end":""},{"old":"/api/v1/stats","type":0,"val":"v1","end":""},{"old":"/api/v1/stats","type":0,"val":"stats","end":""}],"meta":{"params":[]},"name":"releases.stats","handler":{"method":"stats","reference":[null,"stats"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/signup","matchers":{},"tokens":[{"old":"/api/v1/auth/signup","type":0,"val":"api","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"signup","end":""}],"meta":{"params":[]},"name":"auth.new_account.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/new_account_controller')","name":"NewAccount"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/login","matchers":{},"tokens":[{"old":"/api/v1/auth/login","type":0,"val":"api","end":""},{"old":"/api/v1/auth/login","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/login","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/login","type":0,"val":"login","end":""}],"meta":{"params":[]},"name":"auth.access_tokens.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/profile","matchers":{},"tokens":[{"old":"/api/v1/account/profile","type":0,"val":"api","end":""},{"old":"/api/v1/account/profile","type":0,"val":"v1","end":""},{"old":"/api/v1/account/profile","type":0,"val":"account","end":""},{"old":"/api/v1/account/profile","type":0,"val":"profile","end":""}],"meta":{"params":[]},"name":"profile.profile.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/profile_controller')","name":"Profile"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/logout","matchers":{},"tokens":[{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],"meta":{"params":[]},"name":"profile.access_tokens.destroy","handler":{"method":"destroy","reference":[null,"destroy"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}}]}
|
||||
@@ -1,11 +0,0 @@
|
||||
node_modules
|
||||
build
|
||||
tmp
|
||||
debug
|
||||
.git
|
||||
.idea
|
||||
.vscode
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
*.log
|
||||
+8
-3
@@ -12,9 +12,14 @@ APP_URL=http://${HOST}:${PORT}
|
||||
# Session
|
||||
SESSION_DRIVER=cookie
|
||||
|
||||
# Database (the rebuild's own)
|
||||
DB_HOST=127.0.0.1
|
||||
DB_PORT=5432
|
||||
DB_USER=usenet_indexer
|
||||
DB_PASSWORD=
|
||||
DB_DATABASE=usenet_rebuild
|
||||
|
||||
#--------------------------------------------------------------------
|
||||
# CORS (configure allowed origins for API access)
|
||||
#--------------------------------------------------------------------
|
||||
# CORS_ORIGIN=http://localhost:5173,http://localhost:3000
|
||||
# Newznab API key (/api?apikey=...); leave empty to allow access without a key
|
||||
NEWZNAB_API_KEY=
|
||||
# CORS_ORIGIN=http://localhost:5173,http://localhost:3000
|
||||
-29
@@ -1,29 +0,0 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
# Node is pinned: on 24.21, jsonschema 1.5.0 (used by Ace to validate commands)
|
||||
# throws "Invalid URL" on $ref, so no ace command starts. See LOG.md.
|
||||
ARG NODE_VERSION=24.16
|
||||
|
||||
# Build stage: dev dependencies, TypeScript compile, native addons.
|
||||
# The full node image ships python3/make/g++, needed to compile yencode.
|
||||
FROM node:${NODE_VERSION}-bookworm AS build
|
||||
WORKDIR /app
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
RUN node ace build \
|
||||
&& cd build \
|
||||
&& npm ci --omit=dev
|
||||
|
||||
# Runtime stage: compiled app with production dependencies only.
|
||||
FROM node:${NODE_VERSION}-bookworm-slim
|
||||
ENV NODE_ENV=production
|
||||
# bsdtar (libarchive) unpacks compressed RAR entries, e.g. NZBs posted inside RARs.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libarchive-tools \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY --from=build /app/build ./
|
||||
USER node
|
||||
# Every service picks its own command, e.g. `node ace worker:fetch`.
|
||||
CMD ["node", "ace", "list"]
|
||||
+2
-6
@@ -1,4 +1,4 @@
|
||||
import { indexEntities } from '@adonisjs/core'
|
||||
import { indexEntities } from '@adonisjs/core/generators'
|
||||
import { defineConfig } from '@adonisjs/core/app'
|
||||
import { generateRegistry } from '@tuyau/core/hooks'
|
||||
|
||||
@@ -53,7 +53,6 @@ export default defineConfig({
|
||||
() => import('@adonisjs/cors/cors_provider'),
|
||||
() => import('@adonisjs/auth/auth_provider'),
|
||||
() => import('#providers/api_provider'),
|
||||
() => import('@adonisjs/static/static_provider')
|
||||
],
|
||||
|
||||
/*
|
||||
@@ -104,10 +103,7 @@ export default defineConfig({
|
||||
| the production build.
|
||||
|
|
||||
*/
|
||||
metaFiles: [{
|
||||
pattern: 'public/**',
|
||||
reloadServer: false,
|
||||
}],
|
||||
metaFiles: [],
|
||||
|
||||
hooks: {
|
||||
init: [
|
||||
|
||||
@@ -1,123 +0,0 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import env from '#start/env'
|
||||
import SearchService from '#services/SearchService'
|
||||
import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
|
||||
|
||||
const LIMITS = { max: 100, default: 50 }
|
||||
|
||||
/** Search functions and the category each defaults to when `cat` is absent. */
|
||||
const SEARCH_FUNCTIONS: Record<string, number | undefined> = {
|
||||
'search': undefined,
|
||||
'tvsearch': 5000,
|
||||
'tv-search': 5000,
|
||||
'movie': 2000,
|
||||
'movie-search': 2000,
|
||||
'music': 3000,
|
||||
'audio': 3000,
|
||||
'book': 7000,
|
||||
}
|
||||
|
||||
/** The configured key, if any (a Secret from .env, or a plain string set at runtime). */
|
||||
function configuredApiKey(): string | undefined {
|
||||
const value: any = env.get('NEWZNAB_API_KEY')
|
||||
const key = typeof value === 'string' ? value : value?.release()
|
||||
return key || undefined
|
||||
}
|
||||
|
||||
function int(value: unknown): number | undefined {
|
||||
return typeof value === 'string' && /^\d+$/.test(value) ? Number(value) : undefined
|
||||
}
|
||||
|
||||
/**
|
||||
* Newznab API (CONCEPT.md D23) for Prowlarr, Sonarr, Radarr and the like:
|
||||
* GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=…
|
||||
*/
|
||||
export default class NewznabController {
|
||||
async handle({ request, response }: HttpContext) {
|
||||
const qs = request.qs()
|
||||
const t = String(qs.t ?? '')
|
||||
const server = this.server(request)
|
||||
|
||||
const xml = (body: string, type = 'application/xml') => {
|
||||
response.header('Content-Type', `${type}; charset=utf-8`)
|
||||
return response.send(body)
|
||||
}
|
||||
|
||||
const apiKey = configuredApiKey()
|
||||
if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
|
||||
|
||||
if (t === 'caps') return xml(capsXml(server, LIMITS))
|
||||
|
||||
if (t === 'get') {
|
||||
const id = int(qs.id)
|
||||
if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
|
||||
const nzb = await SearchService.nzb(id)
|
||||
if (!nzb) return xml(errorXml(300, 'No such item'))
|
||||
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
|
||||
response.header('Content-Disposition', `attachment; filename="${filename}"`)
|
||||
return xml(nzb.xml, 'application/x-nzb')
|
||||
}
|
||||
|
||||
if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
|
||||
|
||||
const requested = String(qs.cat ?? '')
|
||||
.split(',')
|
||||
.map((c) => int(c.trim()))
|
||||
.filter((c): c is number => c !== undefined)
|
||||
const fallback = SEARCH_FUNCTIONS[t]
|
||||
const categories = expandCategories(requested.length ? requested : fallback ? [fallback] : [])
|
||||
const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max)
|
||||
const offset = int(qs.offset) ?? 0
|
||||
|
||||
// ID-only lookups (tvdbid, imdbid, …) are not supported: caps says so, but
|
||||
// answer with an empty result rather than everything.
|
||||
const q = typeof qs.q === 'string' ? qs.q.trim() : ''
|
||||
const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p])
|
||||
|
||||
const result = idOnly
|
||||
? { total: 0, releases: [] }
|
||||
: await SearchService.search({
|
||||
q,
|
||||
categories,
|
||||
readable: true,
|
||||
notIncomplete: true,
|
||||
hideContainers: true,
|
||||
maxAgeDays: int(qs.maxage),
|
||||
season: int(qs.season),
|
||||
episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
|
||||
sort: 'newest',
|
||||
perPage: limit,
|
||||
offset,
|
||||
})
|
||||
|
||||
const items = result.releases.map((r: any) => {
|
||||
const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name)
|
||||
return {
|
||||
id: r.id,
|
||||
title: r.name,
|
||||
category: r.category ?? 8010,
|
||||
size: r.size,
|
||||
files: r.fileCount,
|
||||
postedAt: r.postedAt ? new Date(r.postedAt) : null,
|
||||
poster: r.poster,
|
||||
groups: r.groups ?? [],
|
||||
// 1 only when a password is needed that we can't give; a known one is in our NZB.
|
||||
password: r.passwordState === 'missing',
|
||||
description: r.spotDescription ?? undefined,
|
||||
season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
|
||||
episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
|
||||
}
|
||||
})
|
||||
return xml(resultsXml(server, items, offset, result.total), 'application/rss+xml')
|
||||
}
|
||||
|
||||
private server(request: HttpContext['request']): NewznabServer {
|
||||
const proto = request.header('x-forwarded-proto') ?? 'http'
|
||||
const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
|
||||
return {
|
||||
title: 'Usenet Indexer',
|
||||
baseUrl: `${proto}://${host}`,
|
||||
apiKey: configuredApiKey(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import SearchService, { type SortOrder } from '#services/SearchService'
|
||||
|
||||
const flag = (value: unknown) => value === '1' || value === 'true' || value === true
|
||||
|
||||
/** A size in MB from the query string, as bytes. */
|
||||
function megabytes(value: unknown): number | undefined {
|
||||
const mb = Number(value)
|
||||
return value !== undefined && value !== '' && Number.isFinite(mb) && mb >= 0 ? Math.round(mb * 1024 * 1024) : undefined
|
||||
}
|
||||
|
||||
export default class ReleasesController {
|
||||
/** GET /api/v1/releases?q=&readable=1&complete=1&hide_encrypted=1&min_size=&max_size=(MB)&ext=mkv,flac&source=&poster=&sort=&page=&per_page= */
|
||||
async index({ request }: HttpContext) {
|
||||
const qs = request.qs()
|
||||
return SearchService.search({
|
||||
q: typeof qs.q === 'string' ? qs.q : undefined,
|
||||
readable: flag(qs.readable),
|
||||
complete: flag(qs.complete),
|
||||
hideEncrypted: flag(qs.hide_encrypted),
|
||||
includePar2Only: flag(qs.par2_only),
|
||||
source: typeof qs.source === 'string' && qs.source ? qs.source : undefined,
|
||||
poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined,
|
||||
minSize: megabytes(qs.min_size),
|
||||
maxSize: megabytes(qs.max_size),
|
||||
extensions: typeof qs.ext === 'string' && qs.ext ? qs.ext.split(/[\s,]+/) : undefined,
|
||||
sort: qs.sort as SortOrder | undefined,
|
||||
page: Number(qs.page) || 1,
|
||||
perPage: Number(qs.per_page) || 50,
|
||||
// The page shows "1,000+"; counting further made filter clicks take seconds.
|
||||
countLimit: 1000,
|
||||
})
|
||||
}
|
||||
|
||||
/** GET /api/v1/releases/:id */
|
||||
async show({ params, response }: HttpContext) {
|
||||
const release = await SearchService.release(Number(params.id))
|
||||
return release ?? response.notFound({ error: 'Release not found' })
|
||||
}
|
||||
|
||||
/** GET /api/v1/releases/:id/nzb */
|
||||
async nzb({ params, response }: HttpContext) {
|
||||
const nzb = await SearchService.nzb(Number(params.id))
|
||||
if (!nzb) return response.notFound({ error: 'Release not found' })
|
||||
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
|
||||
response.header('Content-Type', 'application/x-nzb')
|
||||
response.header('Content-Disposition', `attachment; filename="${filename}"`)
|
||||
return response.send(nzb.xml)
|
||||
}
|
||||
|
||||
/** GET /api/v1/releases/:id/image: the image of the release's spot, read from Usenet. */
|
||||
async image({ params, response }: HttpContext) {
|
||||
const image = await SearchService.spotImage(Number(params.id))
|
||||
if (!image) return response.notFound({ error: 'No image' })
|
||||
response.header('Content-Type', image.type)
|
||||
// A spot's image never changes.
|
||||
response.header('Cache-Control', 'public, max-age=604800, immutable')
|
||||
return response.send(image.data)
|
||||
}
|
||||
|
||||
/** GET /api/v1/stats */
|
||||
async stats() {
|
||||
return SearchService.stats()
|
||||
}
|
||||
}
|
||||
@@ -1,156 +0,0 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import SeedOrchestrator, { LeaseLost } from '#services/SeedOrchestrator'
|
||||
import SeedFetchPool from '#services/SeedFetchPool'
|
||||
import RemoteContentService, { ArticleRelayOff, JobLost } from '#services/RemoteContentService'
|
||||
import type { SeedBatch } from '#lib/seed/wire'
|
||||
|
||||
/**
|
||||
* The seed API (CONCEPT.md D38). Nodes make outgoing HTTPS requests only:
|
||||
* lease a range, post batches of what they found, then finish or release
|
||||
* the range. Bodies may be gzip-compressed (Content-Encoding: gzip).
|
||||
*/
|
||||
export default class SeedController {
|
||||
/** POST /api/v1/seed/lease → 200 { range } or 204 when there's nothing to do. */
|
||||
async lease({ seedNode, response }: HttpContext) {
|
||||
const range = await SeedOrchestrator.lease(seedNode.id)
|
||||
return range ? { range } : response.noContent()
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/ranges/:id/batches with a SeedBatch. */
|
||||
async batch({ seedNode, params, request, response }: HttpContext) {
|
||||
const batch = request.body() as SeedBatch
|
||||
if (!Number.isSafeInteger(batch?.next) || !Array.isArray(batch.files) || !Array.isArray(batch.fragments)) {
|
||||
return response.badRequest({ error: 'Expected { next, headers, files[], fragments[] }' })
|
||||
}
|
||||
return this.leased(response, async () => {
|
||||
const stored = await SeedOrchestrator.batch(seedNode.id, Number(params.id), {
|
||||
...batch,
|
||||
headers: Number(batch.headers) || 0,
|
||||
obfuscated: Number(batch.obfuscated) || 0,
|
||||
deferred: Number(batch.deferred) || 0,
|
||||
})
|
||||
return { stored }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/v1/seed/ranges/:id/chunk?from=&to= → the overviews of those
|
||||
* articles as a zlib/gzip stream (relay nodes fetch through the indexer).
|
||||
*/
|
||||
async chunk({ seedNode, params, request, response }: HttpContext) {
|
||||
if (SeedFetchPool.size === 0) return response.serviceUnavailable({ error: 'Relay is off (SEED_NNTP_CONNECTIONS=0); fetch directly' })
|
||||
const from = Number(request.input('from'))
|
||||
const to = Number(request.input('to'))
|
||||
if (!Number.isSafeInteger(from) || !Number.isSafeInteger(to)) return response.badRequest({ error: 'Expected ?from=&to=' })
|
||||
return this.leased(response, async () => {
|
||||
try {
|
||||
const raw = await SeedOrchestrator.chunk(seedNode.id, Number(params.id), from, to)
|
||||
response.header('content-type', 'application/octet-stream')
|
||||
return response.send(raw)
|
||||
} catch (error) {
|
||||
if (error instanceof RangeError) return response.badRequest({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/ranges/:id/done */
|
||||
async done({ seedNode, params, response }: HttpContext) {
|
||||
return this.leased(response, async () => {
|
||||
await SeedOrchestrator.finish(seedNode.id, Number(params.id))
|
||||
return { ok: true }
|
||||
})
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/ranges/:id/release { error?, uncounted? } */
|
||||
async release({ seedNode, params, request, response }: HttpContext) {
|
||||
const error = request.input('error')
|
||||
const uncounted = request.input('uncounted') === true
|
||||
return this.leased(response, async () => {
|
||||
await SeedOrchestrator.release(seedNode.id, Number(params.id), typeof error === 'string' ? error : undefined, uncounted)
|
||||
return { ok: true }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/v1/seed/inspect/lease { max } → 200 { jobs } or 204: content
|
||||
* inspection and archive walks for the node (CONCEPT.md D41).
|
||||
*/
|
||||
async inspectLease({ seedNode, request, response }: HttpContext) {
|
||||
const max = Math.min(10, Math.max(1, Number(request.input('max')) || 1))
|
||||
const jobs = await RemoteContentService.lease(seedNode.id, max)
|
||||
return jobs.length > 0 ? { jobs } : response.noContent()
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/v1/seed/inspect/done { queue, jobId, token, result } where
|
||||
* result is the probe as a JSON string (the body parser would turn the
|
||||
* empty strings inside it into nulls).
|
||||
*/
|
||||
async inspectDone({ request, response }: HttpContext) {
|
||||
const { queue, jobId, token, result } = request.only(['queue', 'jobId', 'token', 'result'])
|
||||
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string' || typeof result !== 'string') {
|
||||
return response.badRequest({ error: 'Expected { queue, jobId, token, result: JSON string }' })
|
||||
}
|
||||
return this.job(response, () => RemoteContentService.finish(queue, jobId, token, JSON.parse(result)))
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/inspect/failed { queue, jobId, token, error } */
|
||||
async inspectFailed({ request, response }: HttpContext) {
|
||||
const { queue, jobId, token, error } = request.only(['queue', 'jobId', 'token', 'error'])
|
||||
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
|
||||
return response.badRequest({ error: 'Expected { queue, jobId, token, error }' })
|
||||
}
|
||||
return this.job(response, () => RemoteContentService.fail(queue, jobId, token, String(error ?? 'unknown error')))
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/inspect/release { queue, jobId, token }: a stopping node gives a job back. */
|
||||
async inspectRelease({ request, response }: HttpContext) {
|
||||
const { queue, jobId, token } = request.only(['queue', 'jobId', 'token'])
|
||||
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
|
||||
return response.badRequest({ error: 'Expected { queue, jobId, token }' })
|
||||
}
|
||||
return this.job(response, () => RemoteContentService.release(queue, jobId, token))
|
||||
}
|
||||
|
||||
/** GET /api/v1/seed/articles?id=<message id> → the raw article body; 410 when the server doesn't have it. */
|
||||
async article({ request, response }: HttpContext) {
|
||||
const id = request.input('id')
|
||||
if (typeof id !== 'string' || id.length === 0 || id.length > 250) return response.badRequest({ error: 'Expected ?id=<message id>' })
|
||||
try {
|
||||
const body = await RemoteContentService.article(id)
|
||||
if (!body) return response.gone({ error: 'No such article' })
|
||||
response.header('content-type', 'application/octet-stream')
|
||||
return response.send(body)
|
||||
} catch (error: any) {
|
||||
if (error instanceof ArticleRelayOff) return response.serviceUnavailable({ error: error.message })
|
||||
return response.badGateway({ error: `NNTP: ${error.message}` })
|
||||
}
|
||||
}
|
||||
|
||||
/** GET /api/v1/seed/status */
|
||||
async status() {
|
||||
return SeedOrchestrator.status()
|
||||
}
|
||||
|
||||
/** 409 when the job's lock is gone (it was given to another node meanwhile). */
|
||||
private async job(response: HttpContext['response'], action: () => Promise<unknown>) {
|
||||
try {
|
||||
await action()
|
||||
return { ok: true }
|
||||
} catch (error) {
|
||||
if (error instanceof JobLost) return response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** 409 when the range was given to another node meanwhile; the node should drop it. */
|
||||
private async leased(response: HttpContext['response'], action: () => Promise<unknown>) {
|
||||
try {
|
||||
return await action()
|
||||
} catch (error) {
|
||||
if (error instanceof LeaseLost) return response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import type { NextFn } from '@adonisjs/core/types/http'
|
||||
import SeedOrchestrator from '#services/SeedOrchestrator'
|
||||
|
||||
declare module '@adonisjs/core/http' {
|
||||
interface HttpContext {
|
||||
/** The seeding node that made this request (CONCEPT.md D38). */
|
||||
seedNode: { id: number; name: string }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed API requests carry a node's bearer token (`node ace seed:token
|
||||
* <name>`); anything else gets 401. The seed API is the only part of the app
|
||||
* reachable from the internet (traefik routes only /api/v1/seed there).
|
||||
*/
|
||||
export default class SeedNodeMiddleware {
|
||||
async handle(ctx: HttpContext, next: NextFn) {
|
||||
const header = ctx.request.header('authorization') ?? ''
|
||||
const token = header.startsWith('Bearer ') ? header.slice(7).trim() : ''
|
||||
const node = token ? await SeedOrchestrator.nodeForToken(token) : null
|
||||
if (!node) return ctx.response.unauthorized({ error: 'Unknown or disabled seed node token' })
|
||||
ctx.seedNode = node
|
||||
return next()
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
import { BaseModel as AdonisBaseModel } from '@adonisjs/lucid/orm'
|
||||
|
||||
export default class BaseModel extends AdonisBaseModel {
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
import { DateTime } from 'luxon'
|
||||
import { column } from '@adonisjs/lucid/orm'
|
||||
import BaseModel from '#models/base_model'
|
||||
import { loadedSegments, storedSegments } from '#services/FileStore'
|
||||
import type { Segment } from '#lib/nzb/index'
|
||||
|
||||
export default class File extends BaseModel {
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column({ columnName: 'file_key' })
|
||||
declare fileKey: string | null
|
||||
|
||||
@column()
|
||||
declare filename: string
|
||||
|
||||
@column()
|
||||
declare poster: string
|
||||
|
||||
@column()
|
||||
declare date: number
|
||||
|
||||
@column()
|
||||
declare parts: number
|
||||
|
||||
/** Stored compressed (see FileStore); the model sees the list. */
|
||||
@column({ prepare: (value: Segment[]) => storedSegments(value), consume: (value: Buffer | null) => loadedSegments(value) })
|
||||
declare segments: Segment[]
|
||||
|
||||
@column({ columnName: 'first_message_id' })
|
||||
declare firstMessageId: string | null
|
||||
|
||||
@column()
|
||||
declare groups: string[]
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { DateTime } from 'luxon'
|
||||
import { column } from '@adonisjs/lucid/orm'
|
||||
import BaseModel from '#models/base_model'
|
||||
|
||||
export default class Group extends BaseModel {
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare name: string
|
||||
|
||||
@column()
|
||||
declare active: boolean
|
||||
|
||||
/** Every article up to here was fetched successfully (see FetchRangeService). */
|
||||
@column({ columnName: 'last_indexed_id' })
|
||||
declare lastIndexedId: bigint
|
||||
|
||||
/** Download bodies of obfuscated articles to identify them (bandwidth-heavy). */
|
||||
@column({ columnName: 'fetch_bodies' })
|
||||
declare fetchBodies: boolean
|
||||
|
||||
/** How far the scheduler has queued fetch jobs. */
|
||||
@column({ columnName: 'last_queued_id' })
|
||||
declare lastQueuedId: bigint
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
}
|
||||
@@ -1,170 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import FileStore from '#services/FileStore'
|
||||
import IngestService from '#services/IngestService'
|
||||
import type { RarInfo } from '#lib/rar/index'
|
||||
import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
|
||||
|
||||
/** Stored per file and release; a listing of thousands of files is not useful. */
|
||||
const MAX_ENTRIES = 500
|
||||
|
||||
export interface ArchiveResult {
|
||||
info: RarInfo
|
||||
releaseId?: number
|
||||
releaseRenamed?: string
|
||||
grouped?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Records what a RAR volume's headers say: the listing on the file, the
|
||||
* release it belongs to (from the shared directory or continued entry, when
|
||||
* the file has none yet), and for first volumes the contents and password
|
||||
* flag on the release. See CONCEPT.md (D19, D21).
|
||||
*/
|
||||
class ArchiveService {
|
||||
async record(file: any, binary: any, info: RarInfo): Promise<ArchiveResult> {
|
||||
const { next: _next, ...listing } = info
|
||||
const stored = { ...listing, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
|
||||
await FileStore.setDetails(Number(file.id), { archive: stored })
|
||||
|
||||
let releaseId = file.release_id ?? binary?.release_id ?? null
|
||||
let grouped = false
|
||||
const identity = rarReleaseIdentity(info.entries)
|
||||
if (!releaseId && identity) {
|
||||
const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
|
||||
{
|
||||
release: identity,
|
||||
poster: binary?.poster ?? file.poster,
|
||||
weakIdentity: true,
|
||||
groups: binary?.groups ?? file.groups ?? [],
|
||||
postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined,
|
||||
},
|
||||
])
|
||||
releaseId = ids.get(identity.key)!
|
||||
grouped = true
|
||||
if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId })
|
||||
await db.from('files').where('id', file.id).update({ release_id: releaseId })
|
||||
}
|
||||
if (!releaseId) return { info }
|
||||
|
||||
const release = await db.from('releases').where('id', releaseId).first()
|
||||
const update: Record<string, unknown> = {}
|
||||
if (info.firstVolume !== false) {
|
||||
update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted)
|
||||
if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries)
|
||||
} else if (info.headersEncrypted) {
|
||||
update.encrypted = true
|
||||
}
|
||||
const better = releaseNameFromContents(info)
|
||||
// A spot's title is kept (CONCEPT.md D29).
|
||||
const renamed =
|
||||
release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined
|
||||
if (renamed) update.name = renamed
|
||||
if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
|
||||
|
||||
if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`)
|
||||
return { info, releaseId, releaseRenamed: renamed, grouped }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A release name from an archive listing: the top-level directory shared by
|
||||
* all entries when it reads like a release name, otherwise the stem of the
|
||||
* largest entry with a readable name (e.g. an archive holding just
|
||||
* `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped.
|
||||
*/
|
||||
export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string | undefined {
|
||||
// Hidden entries name nothing: one poster adds the same 22 GB ".1917-….mkv" decoy to every upload (LOG.md 2026-10-06 (2)).
|
||||
const listed = info.entries.filter((e) => !isHidden(e.name))
|
||||
const entries = listed.filter((e) => !e.directory)
|
||||
const tops = new Set(listed.map((e) => e.name.split(/[\\/]/)[0]))
|
||||
if (tops.size === 1 && listed.some((e) => /[\\/]/.test(e.name))) {
|
||||
const [top] = tops
|
||||
if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top
|
||||
}
|
||||
const candidates = entries
|
||||
.map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 }))
|
||||
.filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem))
|
||||
.filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem))
|
||||
.sort((a, b) => b.size - a.size)
|
||||
return candidates[0]?.stem
|
||||
}
|
||||
|
||||
/** An entry in a hidden directory or a hidden file itself (`.x/…`, `dir/.x`). */
|
||||
export function isHidden(name: string): boolean {
|
||||
return name.split(/[\\/]/).some((part) => part.startsWith('.'))
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases named after a hidden archive entry before those were skipped:
|
||||
* renamed from their stored listing (no downloads). Used by
|
||||
* `release:rename --hidden`.
|
||||
*/
|
||||
export async function renameHiddenNamed(): Promise<number> {
|
||||
const releases = await db
|
||||
.from('releases')
|
||||
.where('name', 'like', '.%')
|
||||
.whereNot('source', 'spot')
|
||||
.whereNotNull('contents')
|
||||
.select('id', 'name', 'contents')
|
||||
let renamed = 0
|
||||
for (const release of releases) {
|
||||
const entries: any[] = Array.isArray(release.contents) ? release.contents : []
|
||||
const fromHidden = entries.some((e) => isHidden(e.name) && fileStem(String(e.name).split(/[\\/]/).pop() ?? '') === release.name)
|
||||
if (!fromHidden) continue
|
||||
const better = releaseNameFromContents({ entries })
|
||||
if (!better || better === release.name) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
renamed++
|
||||
}
|
||||
return renamed
|
||||
}
|
||||
|
||||
/**
|
||||
* Renames random-named releases from their stored archive listing (no
|
||||
* downloads). Used by `release:rename`.
|
||||
*/
|
||||
export async function renameFromStoredContents(): Promise<number> {
|
||||
const releases = await db
|
||||
.from('releases')
|
||||
.whereNotNull('contents')
|
||||
.whereRaw("jsonb_array_length(contents) > 0")
|
||||
.select('id', 'name', 'source', 'contents')
|
||||
let renamed = 0
|
||||
for (const release of releases) {
|
||||
if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue
|
||||
const better = releaseNameFromContents({ entries: release.contents })
|
||||
if (!better || better === release.name) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
renamed++
|
||||
}
|
||||
return renamed
|
||||
}
|
||||
|
||||
/**
|
||||
* Obfuscated names: one long token of mixed letters and digits, no
|
||||
* separators, and the other generated shapes seen on Usenet (LOG.md
|
||||
* 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
|
||||
*/
|
||||
export function looksRandom(name: string): boolean {
|
||||
// An unknown short extension (e.g. ".bin") says nothing about the name.
|
||||
name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||||
// VIDEO_TS, Sample, CD1: real words, but no name.
|
||||
if (isGenericName(name)) return true
|
||||
if (/^[0-9a-f]{16,}$/i.test(name)) return true
|
||||
// 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
|
||||
if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
|
||||
// UUIDs
|
||||
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
|
||||
// 1650777167 (yEncBin), "83111763 n" (rclone uploads)
|
||||
if (/^\d{6,}( n)?$/.test(name)) return true
|
||||
// prvjr7hapd0i1ny14pz4, and rclone's long base32 names
|
||||
if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
|
||||
// wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
|
||||
if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
|
||||
if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
|
||||
// Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
|
||||
return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
|
||||
}
|
||||
|
||||
export default new ArchiveService()
|
||||
@@ -1,160 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import env from '#start/env'
|
||||
import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import { segmentsOf } from '#services/FileStore'
|
||||
import { looksRandom, releaseNameFromContents } from '#services/ArchiveService'
|
||||
import { rarVolume } from '#lib/content/index'
|
||||
import type { RarEntry, RarInfo } from '#lib/rar/index'
|
||||
import { probeWalk, type WalkProbe, type WalkTask } from '#lib/content/probe'
|
||||
|
||||
/** Stored per release, as for a first-volume listing (ArchiveService). */
|
||||
const MAX_ENTRIES = 500
|
||||
|
||||
/** Listed sizes below this share of the release's size: the first segment listed only part of it. */
|
||||
const COVERED = 0.7
|
||||
|
||||
/** Inspection queues a walk this long after the first volume, when the release has had time to complete. */
|
||||
const WALK_DELAY_MS = 30 * 60 * 1000
|
||||
|
||||
export interface WalkOutcome {
|
||||
releaseId: number
|
||||
/** Nothing downloaded: why. */
|
||||
skipped?: 'missing' | 'covered' | 'volumes'
|
||||
entries?: number
|
||||
before?: number
|
||||
complete?: boolean
|
||||
stopped?: string
|
||||
segments?: number
|
||||
renamed?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete archive listings by walking RAR headers (CONCEPT.md D40). The
|
||||
* first segment of a first volume lists the entries that start in it: for
|
||||
* a set of many small files (an audiobook's MP3s, a music album) that is
|
||||
* the first one or two. The walk reads one segment per further entry and
|
||||
* volume, at low priority: its own queue, one job at a time per collection
|
||||
* worker, and NNTP connections only when inspection isn't waiting for one.
|
||||
*/
|
||||
class ArchiveWalkService {
|
||||
/** Segments one walk may download (~0.7 MB each). */
|
||||
readonly maxSegments = env.get('ARCHIVE_WALK_MAX_SEGMENTS', 400)
|
||||
|
||||
/** A listing that leaves most of the release unaccounted for. */
|
||||
worthWalking(entries: number, listedBytes: number, releaseSize: number): boolean {
|
||||
if (entries === 0 || entries >= MAX_ENTRIES) return false
|
||||
return listedBytes < COVERED * releaseSize
|
||||
}
|
||||
|
||||
/** After inspection listed a first RAR volume whose listing ends inside the set. */
|
||||
async queueAfterInspection(releaseId: number | null, info: RarInfo) {
|
||||
if (!releaseId || info.format === '7z' || info.firstVolume === false) return
|
||||
if (!info.truncated || info.headersEncrypted || info.entries.length === 0) return
|
||||
await QueueService.archiveWalkQueue.add('walk', { releaseId }, { jobId: `walk-${releaseId}`, delay: WALK_DELAY_MS })
|
||||
}
|
||||
|
||||
/**
|
||||
* The database half before a walk (D41): the release's volumes, or the
|
||||
* outcome when there's nothing to walk.
|
||||
*/
|
||||
async prepare(releaseId: number): Promise<{ task: WalkTask } | { outcome: WalkOutcome }> {
|
||||
const release = await db.from('releases').where('id', releaseId).select('id', 'size', 'contents').first()
|
||||
if (!release) return { outcome: { releaseId, skipped: 'missing' } }
|
||||
const before = Array.isArray(release.contents) ? release.contents : []
|
||||
const listed = before.reduce((sum: number, e: any) => sum + Number(e.size ?? 0), 0)
|
||||
if (!this.worthWalking(before.length, listed, Number(release.size))) {
|
||||
return { outcome: await this.done(releaseId, { releaseId, skipped: 'covered' }) }
|
||||
}
|
||||
const volumes = await this.volumes(releaseId)
|
||||
if (!volumes) return { outcome: await this.done(releaseId, { releaseId, skipped: 'volumes' }) }
|
||||
return {
|
||||
task: {
|
||||
kind: 'walk',
|
||||
releaseId,
|
||||
volumes: volumes.map((file) => segmentsOf(file).map(({ number, id }) => ({ number, id }))),
|
||||
maxSegments: this.maxSegments,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** The database half after a walk: the fuller listing, the password flag and a better name. */
|
||||
async apply(releaseId: number, walked: WalkProbe): Promise<WalkOutcome> {
|
||||
const release = await db.from('releases').where('id', releaseId).select('id', 'name', 'source', 'contents', 'encrypted').first()
|
||||
if (!release) return { releaseId, skipped: 'missing' }
|
||||
const before = Array.isArray(release.contents) ? release.contents : []
|
||||
const outcome: WalkOutcome = {
|
||||
releaseId,
|
||||
before: before.length,
|
||||
entries: walked.entries.length,
|
||||
complete: walked.complete,
|
||||
stopped: walked.stopped,
|
||||
segments: walked.segments,
|
||||
}
|
||||
if (walked.entries.length > before.length) {
|
||||
const contents = walked.entries.slice(0, MAX_ENTRIES)
|
||||
const update: Record<string, unknown> = {
|
||||
contents: JSON.stringify(contents),
|
||||
encrypted: Boolean(release.encrypted) || walked.headersEncrypted || walked.entries.some((e) => e.encrypted),
|
||||
}
|
||||
const better = releaseNameFromContents({ entries: walked.entries as RarEntry[] })
|
||||
// The rule of ArchiveService.record: random and post-subject names give way, a spot's title stays.
|
||||
if (better && better !== release.name && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post')) {
|
||||
update.name = better
|
||||
outcome.renamed = better
|
||||
}
|
||||
await db.from('releases').where('id', releaseId).update(update)
|
||||
}
|
||||
return this.done(releaseId, outcome)
|
||||
}
|
||||
|
||||
/** Walks a release here, on the inspection pool's connections when inspection isn't waiting for one. */
|
||||
async walk(releaseId: number): Promise<WalkOutcome> {
|
||||
const prepared = await this.prepare(releaseId)
|
||||
if ('outcome' in prepared) return prepared.outcome
|
||||
return this.apply(releaseId, await probeWalk(prepared.task, fetchLow))
|
||||
}
|
||||
|
||||
private async done(releaseId: number, outcome: WalkOutcome) {
|
||||
await db.from('releases').where('id', releaseId).update({ contents_walked_at: new Date() })
|
||||
return outcome
|
||||
}
|
||||
|
||||
/**
|
||||
* The release's RAR volumes in order, from volume 1 up to the first gap.
|
||||
* Null without a first volume (e.g. volumes with random names).
|
||||
*/
|
||||
private async volumes(releaseId: number): Promise<any[] | null> {
|
||||
const files = await db.from('files').where('release_id', releaseId).select('id', 'filename', 'real_name', 'segments')
|
||||
const sets = new Map<string, Map<number, any>>()
|
||||
for (const file of files) {
|
||||
const volume = rarVolume(String(file.real_name ?? file.filename))
|
||||
if (!volume || segmentsOf(file).length === 0) continue
|
||||
const set = sets.get(volume.base) ?? new Map()
|
||||
if (!set.has(volume.number)) set.set(volume.number, file)
|
||||
sets.set(volume.base, set)
|
||||
}
|
||||
// A release holds one set, now and then a second (a sample, subs): the biggest.
|
||||
const set = [...sets.values()].sort((a, b) => b.size - a.size)[0]
|
||||
if (!set?.has(1)) return null
|
||||
const ordered: any[] = []
|
||||
for (let n = 1; set.has(n); n++) ordered.push(set.get(n))
|
||||
return ordered
|
||||
}
|
||||
}
|
||||
|
||||
/** One article body on a pooled connection, behind inspection's waiting jobs. */
|
||||
async function fetchLow(messageId: string): Promise<Buffer> {
|
||||
const conn = await NntpService.acquire(true)
|
||||
let failed = false
|
||||
try {
|
||||
return await NntpService.timed(NntpService.fetchBody(conn, messageId), 'BODY')
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
export default new ArchiveWalkService()
|
||||
@@ -1,112 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import FetchRangeService from '#services/FetchRangeService'
|
||||
import { articleAtDate } from '#lib/nntp/date_search'
|
||||
|
||||
/** Articles XOVERed to find a posting date near an article number. */
|
||||
const PROBE = 200n
|
||||
|
||||
/**
|
||||
* Header backfill (CONCEPT.md D32). `start` gives a group a cursor from
|
||||
* where live indexing began back to the article posted N days ago; `tick`
|
||||
* (every scheduler run) queues low-priority, header-only fetch ranges from
|
||||
* that cursor downwards, newest first, while few are waiting and the
|
||||
* database stays within its size limits.
|
||||
*/
|
||||
class BackfillService {
|
||||
/** Sets the group's backfill to reach back `days` days. Returns its range. */
|
||||
async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> {
|
||||
const group = await db.from('groups').where('name', groupName).first()
|
||||
if (!group) throw new Error(`Unknown group ${groupName}`)
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`)
|
||||
const dateOf = async (n: bigint) => {
|
||||
const { overviews } = await NntpService.timed<any>(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`)
|
||||
const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
|
||||
const date = header ? Date.parse(header.date) : Number.NaN
|
||||
return Number.isNaN(date) ? null : date
|
||||
}
|
||||
const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf)
|
||||
// Live indexing began a little before the group was added: continue from there.
|
||||
const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf)
|
||||
const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began
|
||||
await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() })
|
||||
return next > until ? { from: until, to: next } : null
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
/** Stops a group's backfill (queued ranges still run). */
|
||||
async stop(groupName: string) {
|
||||
await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null })
|
||||
}
|
||||
|
||||
/** Queues the next backfill ranges. Returns how many. */
|
||||
async tick(): Promise<number> {
|
||||
const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill
|
||||
const groups = await db
|
||||
.from('groups')
|
||||
.where('active', true)
|
||||
.whereNotNull('backfill_until_id')
|
||||
.whereRaw('backfill_next_id > backfill_until_id')
|
||||
.orderBy('id')
|
||||
if (groups.length === 0) return 0
|
||||
|
||||
const room = await this.room(maxDefaultGb, maxHddGb)
|
||||
if (room !== true) {
|
||||
logger.warn(`Backfill paused: ${room}.`)
|
||||
return 0
|
||||
}
|
||||
const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count')
|
||||
let free = maxQueued - Number(count)
|
||||
let queued = 0
|
||||
// Round-robin over the groups, one range each per pass.
|
||||
while (free > 0 && groups.length > 0) {
|
||||
for (const group of [...groups]) {
|
||||
if (free <= 0) break
|
||||
const next = BigInt(group.backfill_next_id)
|
||||
const until = BigInt(group.backfill_until_id)
|
||||
const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n
|
||||
await FetchRangeService.queue(group, start, next, { backfill: true })
|
||||
group.backfill_next_id = (start - 1n).toString()
|
||||
await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id })
|
||||
free--
|
||||
queued++
|
||||
if (start - 1n <= until) {
|
||||
groups.splice(groups.indexOf(group), 1)
|
||||
logger.info(`${group.name}: backfill fully queued.`)
|
||||
}
|
||||
}
|
||||
}
|
||||
return queued
|
||||
}
|
||||
|
||||
/** True when both parts of the database are under their limits, else why not. */
|
||||
/**
|
||||
* Whether the database has room for more history: the default tablespace
|
||||
* (NVMe) and tablespace `hdd` against BACKFILL_MAX_NVME_GB/_HDD_GB. Seed
|
||||
* leases (CONCEPT.md D38) check the same limits.
|
||||
*/
|
||||
async room(
|
||||
maxDefaultGb = indexerConfig.backfill.maxDefaultGb,
|
||||
maxHddGb = indexerConfig.backfill.maxHddGb
|
||||
): Promise<true | string> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')`
|
||||
)
|
||||
const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3
|
||||
if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})`
|
||||
if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})`
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
export default new BackfillService()
|
||||
@@ -1,144 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import FileStore from '#services/FileStore'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
|
||||
export interface BinaryFilesResult {
|
||||
/** New files. */
|
||||
saved: number
|
||||
/** Binaries whose file already existed (same first segment or file_key). */
|
||||
existing: number
|
||||
/** Conflicted, gone or already linked: nothing to do. */
|
||||
skipped: number
|
||||
/** Not all parts stored yet (the job is retried for these). */
|
||||
incomplete: number[]
|
||||
inspections: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns completed binaries into files (worker:file), a batch at a time
|
||||
* (LOG.md 2026-10-06 (7)): one query each for the binaries, their parts and
|
||||
* files that exist already, one INSERT for the new files (so the files
|
||||
* statement trigger updates each release once), and one UPDATE and DELETE
|
||||
* for the binaries' links and copied parts. It was one job of ~8 queries per
|
||||
* binary, ~700 a minute.
|
||||
*/
|
||||
class BinaryFileService {
|
||||
async store(binaryIds: number[]): Promise<BinaryFilesResult> {
|
||||
const result: BinaryFilesResult = { saved: 0, existing: 0, skipped: 0, incomplete: [], inspections: 0 }
|
||||
const ids = [...new Set(binaryIds)]
|
||||
const binaries = await db
|
||||
.from('binaries')
|
||||
.whereIn('id', ids)
|
||||
.whereNull('conflicted_at')
|
||||
.whereNull('file_id')
|
||||
.select('id', 'file_key', 'name', 'real_name', 'poster', 'posted_at', 'groups', 'release_id', 'total_parts')
|
||||
result.skipped = ids.length - binaries.length
|
||||
if (binaries.length === 0) return result
|
||||
|
||||
const partsOf = new Map<number, { number: number; id: string; size: number }[]>()
|
||||
const parts = await db
|
||||
.from('parts')
|
||||
.whereIn('binary_id', binaries.map((b) => b.id))
|
||||
.orderBy([{ column: 'binary_id' }, { column: 'number' }])
|
||||
.select('binary_id', 'number', 'message_id', 'bytes')
|
||||
for (const part of parts) {
|
||||
const list = partsOf.get(Number(part.binary_id)) ?? []
|
||||
list.push({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) })
|
||||
partsOf.set(Number(part.binary_id), list)
|
||||
}
|
||||
const complete = binaries.filter((b) => {
|
||||
const ok = (partsOf.get(Number(b.id))?.length ?? 0) >= b.total_parts
|
||||
if (!ok) result.incomplete.push(Number(b.id))
|
||||
return ok
|
||||
})
|
||||
if (complete.length === 0) return result
|
||||
|
||||
// Already known as a file, e.g. added from a posted NZB before its own articles completed (id only: the row holds the segment list).
|
||||
const firsts = complete.map((b) => partsOf.get(Number(b.id))![0].id)
|
||||
const byFirst = new Map<string, number>()
|
||||
for (const row of await db.from('files').whereIn('first_message_id', firsts).select('id', 'first_message_id')) {
|
||||
byFirst.set(row.first_message_id, Number(row.id))
|
||||
}
|
||||
const fileIdOf = new Map<number, number>()
|
||||
const fresh = complete.filter((b) => {
|
||||
const existing = byFirst.get(partsOf.get(Number(b.id))![0].id)
|
||||
if (existing) fileIdOf.set(Number(b.id), existing)
|
||||
return !existing
|
||||
})
|
||||
|
||||
const now = new Date()
|
||||
// file_key is unique: a second completion of the same upload inserts nothing.
|
||||
const inserted = await FileStore.insert(
|
||||
fresh.map((b) => {
|
||||
const segments = partsOf.get(Number(b.id))!
|
||||
return {
|
||||
file_key: b.file_key,
|
||||
filename: b.name,
|
||||
poster: b.poster,
|
||||
date: b.posted_at ? new Date(b.posted_at).getTime() : now.getTime(),
|
||||
parts: segments.length,
|
||||
segments,
|
||||
groups: JSON.stringify(b.groups),
|
||||
release_id: b.release_id,
|
||||
real_name: b.real_name,
|
||||
size: segments.reduce((sum, s) => sum + s.size, 0),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
})
|
||||
)
|
||||
const insertedByKey = new Map(inserted.map((row) => [row.file_key, row.id]))
|
||||
const missing = fresh.filter((b) => !insertedByKey.has(b.file_key)).map((b) => b.file_key)
|
||||
const byKey = new Map<string, number>()
|
||||
if (missing.length > 0) {
|
||||
for (const row of await db.from('files').whereIn('file_key', missing).select('id', 'file_key')) byKey.set(row.file_key, Number(row.id))
|
||||
}
|
||||
for (const b of fresh) {
|
||||
const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key)
|
||||
if (id) fileIdOf.set(Number(b.id), id)
|
||||
}
|
||||
const fresher = new Map(fresh.map((b) => [b.file_key, b]))
|
||||
result.saved = inserted.length
|
||||
result.existing = fileIdOf.size - inserted.length
|
||||
|
||||
// Links, locked in file_key order like ingest's upsert, so the two can't deadlock.
|
||||
const linked = complete.filter((b) => fileIdOf.has(Number(b.id))).sort((a, b) => a.file_key.localeCompare(b.file_key))
|
||||
if (linked.length > 0) {
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(
|
||||
`SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`,
|
||||
[linked.map((b) => b.id)]
|
||||
)
|
||||
// Files queued for inspection below keep their binary until it has its fingerprint (par2 naming, D16).
|
||||
const queued = new Set(inserted.filter((row) => row.file_key && ContentService.isCandidate(String(fresher.get(row.file_key)?.name ?? ''))).map((row) => row.id))
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET file_id = v.file_id, awaiting_inspection = CASE WHEN v.queued THEN true ELSE b.awaiting_inspection END
|
||||
FROM unnest(?::bigint[], ?::bigint[], ?::boolean[]) AS v(id, file_id, queued) WHERE b.id = v.id`,
|
||||
[linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!), linked.map((b) => queued.has(fileIdOf.get(Number(b.id))!))]
|
||||
)
|
||||
})
|
||||
// Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20).
|
||||
await db.rawQuery(`DELETE FROM parts WHERE binary_id = ANY(?::bigint[]) AND pcrc32 IS NULL`, [linked.map((b) => b.id)])
|
||||
}
|
||||
|
||||
// As for one file before: an SFV posted earlier may list them (D34), then inspection.
|
||||
const added = inserted.flatMap((row) => {
|
||||
const b = row.file_key ? fresher.get(row.file_key) : undefined
|
||||
return b
|
||||
? [{ id: row.id, name: b.real_name ?? b.name, posted: String(b.name), poster: b.poster, date: b.posted_at ? new Date(b.posted_at).getTime() : null, release_id: b.release_id }]
|
||||
: []
|
||||
})
|
||||
if (added.length > 0) await SfvService.claimAll(added.map(({ posted: _p, ...file }) => file))
|
||||
// By the posted name, as before: inspection skips later volumes and readable media names anyway.
|
||||
const jobs = added
|
||||
.filter((file) => ContentService.isCandidate(file.posted))
|
||||
.map((file) => file.id)
|
||||
if (jobs.length > 0) await InspectionQueue.add(jobs)
|
||||
result.inspections = jobs.length
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
export default new BinaryFileService()
|
||||
@@ -1,321 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NntpService from '#services/NntpService'
|
||||
import Par2Service, { type Par2Result } from '#services/Par2Service'
|
||||
import ArchiveService, { looksRandom, type ArchiveResult } from '#services/ArchiveService'
|
||||
import NameService from '#services/NameService'
|
||||
import { decodeSegment } from '#lib/yenc/index'
|
||||
import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
|
||||
import {
|
||||
isArchiveStart,
|
||||
isLaterVolume,
|
||||
NICOVIDEO_FILE,
|
||||
NICOVIDEO_RELEASE,
|
||||
nicovideoReleaseName,
|
||||
type ContentType,
|
||||
type NicovideoInfo,
|
||||
} from '#lib/content/index'
|
||||
import { chunkMd5, probeInspection, type InspectProbe, type InspectTask, type TaskSegment } from '#lib/content/probe'
|
||||
import type { Segment } from '#lib/nzb/index'
|
||||
import { fileStem, releaseIdentity } from '#lib/release/index'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { looksLikeFilename } from '#lib/subject/index'
|
||||
import { sevenZipVolume } from '#lib/sevenzip/index'
|
||||
import SevenZipService from '#services/SevenZipService'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ArchiveWalkService from '#services/ArchiveWalkService'
|
||||
import FileStore, { segmentsOf } from '#services/FileStore'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
|
||||
/**
|
||||
* MD5 of the first 16 KiB of a file, as par2 records it, from the body of
|
||||
* the file's first part. Undefined when that part is shorter than 16 KiB but
|
||||
* the file is not (the chunk would span two parts).
|
||||
*/
|
||||
export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined {
|
||||
return chunkMd5(decodeSegment(body).data, fileSize)
|
||||
}
|
||||
|
||||
export interface ContentResult {
|
||||
par2?: Par2Result
|
||||
archive?: ArchiveResult
|
||||
named: number
|
||||
contentType: ContentType | null
|
||||
nzb?: NzbImportResult
|
||||
/** NZBs read from inside an archive. */
|
||||
archiveNzbs?: NzbImportResult[]
|
||||
/** Files an SFV's listing moved into its release. */
|
||||
sfvMoved?: number
|
||||
/** A niconico capture's release name from its API response (D34). */
|
||||
nicovideo?: string
|
||||
/** The real name from the yEnc header, for a file posted under a random subject (D36). */
|
||||
yencName?: string
|
||||
/** Why the file's release was deleted as garbage after this probe (D42). */
|
||||
garbage?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes names and groupings from file contents (CONCEPT.md D21). For a
|
||||
* completed file with a random name, one article (the first) is enough:
|
||||
* - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
|
||||
* - par2 files are recognised by content and name every file of their set,
|
||||
* - RAR headers give the release grouping, name and contents.
|
||||
* Inspection is prepare (database) → probe (downloads and parsing,
|
||||
* lib/content/probe; here or on a seeding node, D41) → apply (database).
|
||||
*/
|
||||
class ContentService {
|
||||
/** Random or missing names, and named first RAR volumes (for contents and password flag). */
|
||||
isCandidate(name: string) {
|
||||
// Even under a random stem: the first volume and the index .par2 are inspected instead.
|
||||
if (isLaterVolume(name)) return false
|
||||
if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
|
||||
if (isArchiveStart(name)) return true
|
||||
if (/\.nzb$/i.test(name)) return true
|
||||
// SFVs name the files of their release; niconico captures carry their title (D34).
|
||||
if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true
|
||||
// Every 7z volume: the set is listed when its last volume is in (see inspect).
|
||||
if (sevenZipVolume(name) !== null) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads that randomise the subject per article often keep the real file
|
||||
* name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`;
|
||||
* LOG.md 2026-10-02 (2)). A file with a random name takes that name, and
|
||||
* leaves its weak one-file release for the release its name gives, where
|
||||
* the other files of the upload gather. Returns the name, or undefined.
|
||||
*/
|
||||
private async nameFromYenc(file: any, binary: any, yencName: string): Promise<string | undefined> {
|
||||
const current = String(file.real_name ?? file.filename)
|
||||
if (file.real_name || !looksRandom(fileStem(current))) return undefined
|
||||
if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined
|
||||
|
||||
const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null
|
||||
const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1)
|
||||
let releaseId = file.release_id
|
||||
if (weak) {
|
||||
const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true })
|
||||
if (identity) {
|
||||
const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
|
||||
{ release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true },
|
||||
])
|
||||
releaseId = ids.get(identity.key) ?? releaseId
|
||||
}
|
||||
}
|
||||
await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId })
|
||||
if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId })
|
||||
return yencName
|
||||
}
|
||||
|
||||
/**
|
||||
* Names a niconico capture's release after the video, while it still has
|
||||
* the archiver's `<time>_nicovideo_jp_watch_sm<id>` name; its tags join
|
||||
* the search text.
|
||||
*/
|
||||
private async nameNicovideo(file: any, info: NicovideoInfo, id: string): Promise<string | undefined> {
|
||||
if (!file.release_id) return undefined
|
||||
const release = await db.from('releases').where('id', file.release_id).first()
|
||||
if (!release || !NICOVIDEO_RELEASE.test(release.name)) return undefined
|
||||
const name = nicovideoReleaseName(info, id)
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', release.id)
|
||||
.update({ name, search_files: info.tags.join(' ').slice(0, 4000) || null })
|
||||
return name
|
||||
}
|
||||
|
||||
/**
|
||||
* The database half before a probe (CONCEPT.md D41): what to inspect for
|
||||
* a file, or null when there's nothing to do. A later 7z volume becomes its
|
||||
* archive's first volume, once all volumes are in.
|
||||
*/
|
||||
async prepare(fileId: number): Promise<InspectTask | null> {
|
||||
const file = await db.from('files').where('id', fileId).first()
|
||||
if (!file || !this.isCandidate(file.filename)) return null
|
||||
// Stored before its signature was found to be garbage (D42): not worth an article.
|
||||
if (await GarbageFilter.skipProbe(file)) return null
|
||||
if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
|
||||
const volumes = await SevenZipService.volumes(file)
|
||||
if (!volumes || (await FileStore.details(Number(volumes[0].id))).archive) return null
|
||||
return this.prepare(Number(volumes[0].id))
|
||||
}
|
||||
const segments = segmentsOf(file)
|
||||
if (!segments[0]?.id) return null
|
||||
// A 7z listing reads the last volume's tail (sniffing decides whether it's 7z: random names too).
|
||||
const volumes = await SevenZipService.volumes(file)
|
||||
const last = volumes && volumes.length > 1 ? segmentsOf(volumes[volumes.length - 1]) : segments
|
||||
return {
|
||||
kind: 'inspect',
|
||||
fileId: Number(file.id),
|
||||
name: String(file.real_name ?? file.filename),
|
||||
segments: taskSegments(segments),
|
||||
sevenZip: volumes ? { volumes: volumes.length, last: taskSegments(last, true) } : null,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The database half after a probe: names, groupings, listings and flags from
|
||||
* what it found. Then the release is deleted if it turned out to be garbage (D42).
|
||||
*/
|
||||
async apply(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
|
||||
let result: ContentResult | undefined
|
||||
// Another probe deleted a release this one writes to as garbage meanwhile (LOG.md 2026-10-08 (7)):
|
||||
// the file's own release (nothing left to do), or one its par2 set joins. Again once that
|
||||
// deletion has committed: the set then finds the release's files gone and makes its own.
|
||||
for (const wait of [0, 1000, 3000]) {
|
||||
if (wait) await new Promise((resolve) => setTimeout(resolve, wait))
|
||||
try {
|
||||
result = await this.applyProbe(task, probe)
|
||||
break
|
||||
} catch (error: any) {
|
||||
if (error.code !== '23503' || wait === 3000) throw error
|
||||
if (!(await db.from('files').where('id', task.fileId).select('id').first())) {
|
||||
return { named: 0, contentType: probe.contentType, garbage: 'deleted meanwhile' }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!result) throw new Error(`Applying the probe of file ${task.fileId} gave no result`)
|
||||
result.garbage = (await GarbageFilter.afterProbe(task.fileId, probe.contentType)) ?? undefined
|
||||
return result
|
||||
}
|
||||
|
||||
private async applyProbe(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
|
||||
const result: ContentResult = { named: 0, contentType: probe.contentType }
|
||||
const file = await db.from('files').where('id', task.fileId).first()
|
||||
if (!file) return result
|
||||
const binary = await db.from('binaries').where('file_id', task.fileId).first()
|
||||
const { contentType } = probe
|
||||
await FileStore.setDetails(Number(file.id), { content_type: contentType })
|
||||
if (contentType === 'missing') {
|
||||
// Nothing to read; the binary needn't wait for inspection any more.
|
||||
if (binary) await db.from('binaries').where('id', binary.id).update({ awaiting_inspection: null })
|
||||
return result
|
||||
}
|
||||
|
||||
// Fingerprint, so par2 sets (now or later) can name this file exactly.
|
||||
if (binary) {
|
||||
await db
|
||||
.from('binaries')
|
||||
.where('id', binary.id)
|
||||
.update({
|
||||
size: db.raw('coalesce(size, ?)', [probe.size]),
|
||||
md5_16k: db.raw('coalesce(md5_16k, ?)', [probe.md5_16k]),
|
||||
// Inspected: the GC may delete it again by its usual rules (LOG.md 2026-10-06 (11)).
|
||||
awaiting_inspection: null,
|
||||
})
|
||||
}
|
||||
|
||||
result.yencName = await this.nameFromYenc(file, binary, probe.yencName ?? '')
|
||||
if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
|
||||
// An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
|
||||
// to read, and useless without the key.
|
||||
const releaseId = file.release_id ?? binary?.release_id
|
||||
if (releaseId) {
|
||||
await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true })
|
||||
}
|
||||
return result
|
||||
}
|
||||
const nicovideo = NICOVIDEO_FILE.exec(task.name)
|
||||
if (nicovideo) {
|
||||
if (probe.nicovideo) result.nicovideo = await this.nameNicovideo(file, probe.nicovideo, nicovideo[1])
|
||||
return result
|
||||
}
|
||||
if (/\.sfv$/i.test(task.name)) {
|
||||
if (typeof probe.text === 'string') result.sfvMoved = (await SfvService.attach(file, probe.text)) ?? undefined
|
||||
return result
|
||||
}
|
||||
if (contentType === 'nzb') {
|
||||
if (typeof probe.text === 'string') result.nzb = (await NzbImportService.import(file, probe.text)) ?? undefined
|
||||
return result
|
||||
}
|
||||
if (probe.par2 && binary) {
|
||||
result.par2 = await Par2Service.attach(binary, probe.par2)
|
||||
} else if (probe.archive && probe.archive.format === '7z') {
|
||||
result.archive = await ArchiveService.record(file, binary, probe.archive)
|
||||
} else if (probe.archive) {
|
||||
result.archive = await ArchiveService.record(file, binary, probe.archive)
|
||||
// The rest of the listing, later and at low priority (D40).
|
||||
await ArchiveWalkService.queueAfterInspection(result.archive.releaseId ?? null, probe.archive)
|
||||
if (probe.archiveNzbs?.length) {
|
||||
result.archiveNzbs = []
|
||||
for (const { entryName, xml } of probe.archiveNzbs) {
|
||||
const imported = await NzbImportService.import(file, xml, { entryName, moveContainer: (probe.archiveNzbEntries ?? 0) === 1 })
|
||||
if (imported) result.archiveNzbs.push(imported)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (binary) result.named = await NameService.resolve([binary.id])
|
||||
return result
|
||||
}
|
||||
|
||||
/** Inspects a file here: prepare, probe on the inspection pool's connections, apply. */
|
||||
async inspect(fileId: number): Promise<ContentResult | null> {
|
||||
const task = await this.prepare(fileId)
|
||||
if (!task) return null
|
||||
const probe = await probeInspection(task, fetchPooled, (message) => logger.warn(message))
|
||||
return this.apply(task, probe)
|
||||
}
|
||||
}
|
||||
|
||||
/** A task's segment list: all of a small file's; of a big one, the first and the last few (a 7z tail). */
|
||||
function taskSegments(segments: Segment[], tailOnly = false): TaskSegment[] {
|
||||
const list = segments.map(({ number, id, size }) => ({ number, id, size }))
|
||||
if (list.length <= TASK_SEGMENTS) return list
|
||||
return tailOnly ? list.slice(-TASK_SEGMENTS) : [list[0], ...list.slice(-TASK_SEGMENTS)]
|
||||
}
|
||||
|
||||
/** More than whole files (40 parts) and par2 files (20) are read; a 7z tail of 16 MB fits easily. */
|
||||
const TASK_SEGMENTS = 40
|
||||
|
||||
/** One article body on a pooled connection. */
|
||||
async function fetchPooled(messageId: string): Promise<Buffer> {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
|
||||
} catch (error: any) {
|
||||
// A missing article (430) leaves the connection fine.
|
||||
failed = !/^4\d\d/.test(String(error.message))
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One line per finding, as worker:collection logs them (and the seed API
|
||||
* for nodes' results).
|
||||
*/
|
||||
export function describeContent(filename: string, content: ContentResult | null): string[] {
|
||||
const lines: string[] = []
|
||||
if (content?.par2) {
|
||||
const { par2 } = content
|
||||
lines.push(`Par2 "${filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
|
||||
} else if (content?.archive) {
|
||||
const { info, releaseRenamed, grouped } = content.archive
|
||||
const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
|
||||
.filter(Boolean)
|
||||
.join(', ')
|
||||
lines.push(
|
||||
`Archive "${filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
|
||||
(grouped ? ', grouped into a release' : '') +
|
||||
(releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
|
||||
)
|
||||
}
|
||||
if (content?.named) lines.push(`"${filename}" got its real name from par2.`)
|
||||
if (content?.contentType === 'missing') lines.push(`"${filename}": the server no longer has its first article.`)
|
||||
if (content?.contentType === 'nzbe') lines.push(`"${filename}" is an encrypted NZB container.`)
|
||||
if (content?.contentType === 'rclone') lines.push(`"${filename}" is rclone-encrypted.`)
|
||||
if (content?.yencName) lines.push(`"${filename}" is "${content.yencName}" (yEnc name).`)
|
||||
if (content?.garbage) lines.push(`"${filename}": release deleted as garbage (${content.garbage === 'deleted meanwhile' ? 'by another probe' : `${content.garbage}, no name or password`}).`)
|
||||
if (content?.nicovideo) lines.push(`"${filename}" → "${content.nicovideo}".`)
|
||||
if (content?.sfvMoved) lines.push(`SFV "${filename}" moved ${content.sfvMoved} files into its release.`)
|
||||
if (content?.contentType === 'yencbin') lines.push(`"${filename}" is a yEncBin encrypted container.`)
|
||||
if (content?.nzb) {
|
||||
const n = content.nzb
|
||||
lines.push(`NZB "${filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
export default new ContentService()
|
||||
@@ -1,264 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import { type OverviewHeader } from '#services/IngestService'
|
||||
import { decodeSegment, inferTotalParts, parseYencMeta } from '#lib/yenc/index'
|
||||
import { sniffContentType } from '#lib/content/index'
|
||||
import { parseRarHeaders } from '#lib/rar/index'
|
||||
|
||||
/** Expanding re-reads the file's article range with XOVER; beyond this it is too costly. */
|
||||
const EXPAND_MAX_RANGE = 200_000
|
||||
/** A claimed row whose worker died is taken again after this long. */
|
||||
const RECLAIM_AFTER = '1 hour'
|
||||
|
||||
export interface DeferredArticle {
|
||||
articleNumber: number
|
||||
header: OverviewHeader
|
||||
}
|
||||
|
||||
/** File name from a deferred uploader's subject: `[i/M] "name" yEnc`, no part counter. */
|
||||
export function deferredName(subject: string | undefined): string | null {
|
||||
return /"([^"]+)"/.exec(subject ?? '')?.[1]?.trim() || null
|
||||
}
|
||||
|
||||
export function isDeferred(header: OverviewHeader): boolean {
|
||||
const id = header['message-id'] ?? ''
|
||||
return indexerConfig.deferBodyMessageIds.some((re) => re.test(id))
|
||||
}
|
||||
|
||||
/** A recorded range never spans more articles than this (one XOVER to expand it). */
|
||||
const RANGE_MAX_SPAN = 50_000
|
||||
|
||||
export function bodyJob(header: OverviewHeader, group: string, articleNumber: number | bigint) {
|
||||
return {
|
||||
name: 'process-body',
|
||||
data: { header, group, articleNumber: Number(articleNumber) },
|
||||
// By message-id: an article crossposted to several indexed groups is downloaded once.
|
||||
opts: { jobId: `body-${createHash('sha1').update(header['message-id'] ?? '').digest('hex')}` },
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deferred uploads (CONCEPT.md D28). Their articles skip the body queue and
|
||||
* are recorded here per file, from the headers alone. `worker:deferred`
|
||||
* later downloads one body per file for its size, part count and content
|
||||
* type, and sends small files (NZBs in RARs, par2 index files) to the body
|
||||
* queue completely.
|
||||
*/
|
||||
class DeferredService {
|
||||
/**
|
||||
* Records one group's deferred articles: per file when the subject names
|
||||
* the file, otherwise only as article ranges. Returns the number of files.
|
||||
*/
|
||||
async record(groupName: string, articles: DeferredArticle[]): Promise<number> {
|
||||
const named = articles.filter((a) => deferredName(a.header.subject) !== null)
|
||||
await this.recordRanges(groupName, articles.filter((a) => deferredName(a.header.subject) === null))
|
||||
return this.recordFiles(groupName, named)
|
||||
}
|
||||
|
||||
private async recordRanges(groupName: string, articles: DeferredArticle[]) {
|
||||
if (articles.length === 0) return
|
||||
const sorted = [...articles].sort((a, b) => Number(a.articleNumber) - Number(b.articleNumber))
|
||||
const now = new Date()
|
||||
const rows: Record<string, unknown>[] = []
|
||||
for (const article of sorted) {
|
||||
const n = Number(article.articleNumber)
|
||||
const bytes = Number.parseInt(article.header[':bytes'] ?? '0', 10) || 0
|
||||
const last = rows.at(-1)
|
||||
if (last && n - (last.first_article as number) <= RANGE_MAX_SPAN) {
|
||||
last.last_article = n
|
||||
;(last.article_count as number)++
|
||||
last.bytes = (last.bytes as number) + bytes
|
||||
} else {
|
||||
rows.push({ group_name: groupName, first_article: n, last_article: n, article_count: 1, bytes, created_at: now })
|
||||
}
|
||||
}
|
||||
await db.table('deferred_ranges').multiInsert(rows)
|
||||
}
|
||||
|
||||
private async recordFiles(groupName: string, articles: DeferredArticle[]): Promise<number> {
|
||||
const files = new Map<string, {
|
||||
name: string; subject: string; poster: string; first: number; firstId: string; last: number
|
||||
count: number; bytes: number; postedAt: Date | null
|
||||
}>()
|
||||
for (const { articleNumber, header } of articles) {
|
||||
const name = deferredName(header.subject)!
|
||||
const n = Number(articleNumber)
|
||||
const bytes = Number.parseInt(header[':bytes'] ?? '0', 10) || 0
|
||||
const date = header.date ? new Date(header.date) : null
|
||||
const postedAt = date && !Number.isNaN(date.getTime()) ? date : null
|
||||
const file = files.get(name)
|
||||
if (!file) {
|
||||
files.set(name, {
|
||||
name, subject: header.subject ?? '', poster: header.from ?? '', first: n,
|
||||
firstId: (header['message-id'] ?? '').replace(/^<|>$/g, ''), last: n, count: 1, bytes, postedAt,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (n < file.first) {
|
||||
file.first = n
|
||||
file.firstId = (header['message-id'] ?? '').replace(/^<|>$/g, '')
|
||||
}
|
||||
file.last = Math.max(file.last, n)
|
||||
file.count++
|
||||
file.bytes += bytes
|
||||
if (postedAt && (!file.postedAt || postedAt < file.postedAt)) file.postedAt = postedAt
|
||||
}
|
||||
if (files.size === 0) return 0
|
||||
|
||||
const now = new Date()
|
||||
const rows = [...files.values()].map((f) => ({
|
||||
file_key: createHash('sha1').update(`deferred\n${f.name}`).digest('hex'),
|
||||
name: f.name,
|
||||
subject: f.subject,
|
||||
poster: f.poster,
|
||||
group_name: groupName,
|
||||
first_article: f.first,
|
||||
first_message_id: f.firstId,
|
||||
last_article: f.last,
|
||||
article_count: f.count,
|
||||
bytes: f.bytes,
|
||||
posted_at: f.postedAt,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
}))
|
||||
// Counts and the article range only add up within the group the file was first seen in.
|
||||
await db.rawQuery(
|
||||
`INSERT INTO deferred_files AS d (${Object.keys(rows[0]).join(', ')})
|
||||
SELECT * FROM jsonb_to_recordset(?::jsonb) AS r(
|
||||
file_key varchar, name text, subject text, poster text, group_name text, first_article bigint,
|
||||
first_message_id text, last_article bigint, article_count int, bytes bigint, posted_at timestamptz,
|
||||
first_seen_at timestamptz, last_seen_at timestamptz)
|
||||
ON CONFLICT (file_key) DO UPDATE SET
|
||||
last_seen_at = EXCLUDED.last_seen_at,
|
||||
first_message_id = CASE WHEN d.group_name = EXCLUDED.group_name AND EXCLUDED.first_article < d.first_article
|
||||
THEN EXCLUDED.first_message_id ELSE d.first_message_id END,
|
||||
first_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN LEAST(d.first_article, EXCLUDED.first_article) ELSE d.first_article END,
|
||||
last_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN GREATEST(d.last_article, EXCLUDED.last_article) ELSE d.last_article END,
|
||||
article_count = d.article_count + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.article_count ELSE 0 END,
|
||||
bytes = d.bytes + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.bytes ELSE 0 END,
|
||||
posted_at = LEAST(d.posted_at, EXCLUDED.posted_at)`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
return files.size
|
||||
}
|
||||
|
||||
/** Takes up to `limit` files that stopped growing; they are marked `processing`. */
|
||||
async claim(limit: number): Promise<any[]> {
|
||||
const settled = new Date(Date.now() - indexerConfig.deferredSettleMs)
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE deferred_files SET status = 'processing', processed_at = now()
|
||||
WHERE id IN (
|
||||
SELECT id FROM deferred_files
|
||||
WHERE (status = 'pending' AND last_seen_at < ?)
|
||||
OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
|
||||
ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[settled, limit]
|
||||
)
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues the bodies of up to `limit` recorded ranges, oldest first: the
|
||||
* articles are found again with XOVER. Returns ranges and articles queued.
|
||||
*/
|
||||
async expandRanges(limit: number): Promise<{ ranges: number; articles: number }> {
|
||||
const ranges = await db.from('deferred_ranges').whereNull('expanded_at').orderBy('id').limit(limit)
|
||||
let articles = 0
|
||||
for (const range of ranges) {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let overviews: [number, OverviewHeader][]
|
||||
try {
|
||||
await conn.group(range.group_name)
|
||||
overviews = (await conn.xover(String(range.first_article), String(range.last_article))).overviews
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
const jobs = overviews
|
||||
.filter(([, header]) => header && isDeferred(header) && deferredName(header.subject) === null)
|
||||
.map(([articleNumber, header]) => bodyJob(header, range.group_name, articleNumber))
|
||||
for (let i = 0; i < jobs.length; i += 5000) await QueueService.bodyQueue.addBulk(jobs.slice(i, i + 5000))
|
||||
await db.from('deferred_ranges').where('id', range.id).update({ expanded_at: new Date() })
|
||||
articles += jobs.length
|
||||
}
|
||||
return { ranges: ranges.length, articles }
|
||||
}
|
||||
|
||||
/** Reads one body of the file; expands small files into the body queue. Returns the new status. */
|
||||
async process(file: any): Promise<string> {
|
||||
const update: Record<string, unknown> = { processed_at: new Date(), error: null }
|
||||
try {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let body: Buffer
|
||||
try {
|
||||
body = await NntpService.fetchBody(conn, file.first_message_id)
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
|
||||
const meta = parseYencMeta(body)
|
||||
const totalParts = inferTotalParts(meta)
|
||||
Object.assign(update, {
|
||||
yenc_name: meta.begin.name,
|
||||
size: meta.begin.size,
|
||||
total_parts: totalParts,
|
||||
sampled_part: meta.begin.part ?? 1,
|
||||
})
|
||||
if ((meta.begin.part ?? 1) === 1) {
|
||||
const data = decodeSegment(body, { lenientSize: true, verifyCrc: false }).data
|
||||
update.content_type = sniffContentType(data)
|
||||
const rar = parseRarHeaders(data)
|
||||
if (rar) update.encrypted = rar.headersEncrypted || rar.entries.some((e) => e.encrypted)
|
||||
}
|
||||
|
||||
update.status = 'sampled'
|
||||
if (totalParts <= indexerConfig.deferredExpandMaxParts && update.encrypted !== true) {
|
||||
const queued = await this.expand(file)
|
||||
if (queued > 0) update.status = 'expanded'
|
||||
}
|
||||
} catch (error: any) {
|
||||
update.status = 'failed'
|
||||
update.error = String(error.message).slice(0, 500)
|
||||
}
|
||||
await db.from('deferred_files').where('id', file.id).update(update)
|
||||
return update.status as string
|
||||
}
|
||||
|
||||
/** Finds the file's articles again (XOVER over its range) and queues their bodies. */
|
||||
private async expand(file: any): Promise<number> {
|
||||
const first = Number(file.first_article)
|
||||
const last = Number(file.last_article)
|
||||
if (last - first > EXPAND_MAX_RANGE) return 0
|
||||
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let overviews: [number, OverviewHeader][]
|
||||
try {
|
||||
await conn.group(file.group_name)
|
||||
overviews = (await conn.xover(String(first), String(last))).overviews
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
const jobs = overviews
|
||||
.filter(([, header]) => header && deferredName(header.subject) === file.name)
|
||||
.map(([articleNumber, header]) => bodyJob(header, file.group_name, articleNumber))
|
||||
if (jobs.length > 0) await QueueService.bodyQueue.addBulk(jobs)
|
||||
return jobs.length
|
||||
}
|
||||
}
|
||||
|
||||
export default new DeferredService()
|
||||
@@ -1,118 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import QueueService from '#services/QueueService'
|
||||
import indexerConfig from '#config/indexer'
|
||||
|
||||
/** A failed range is retried after this long... */
|
||||
const REQUEUE_AFTER_MS = 5 * 60 * 1000
|
||||
/** ...at most this many times; after that it stays visible as a gap. */
|
||||
const MAX_REQUEUES = 5
|
||||
/** A queued range without a job in Redis (e.g. Redis was flushed) is re-queued after this long. */
|
||||
const LOST_AFTER_MS = 60 * 60 * 1000
|
||||
|
||||
interface Group {
|
||||
id: number
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks the article ranges given to worker:fetch. A range only counts as
|
||||
* indexed once the worker reports it done, so failed or lost fetch jobs are
|
||||
* retried instead of silently skipped. See CONCEPT.md (D18).
|
||||
*/
|
||||
class FetchRangeService {
|
||||
/** Queues a range. A backfill range is header-only and waits behind live ones (CONCEPT.md D32). */
|
||||
async queue(group: Group, startId: bigint, endId: bigint, { backfill = false } = {}) {
|
||||
const now = new Date()
|
||||
const [row] = await db
|
||||
.table('fetch_ranges')
|
||||
.insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), backfill, created_at: now, updated_at: now })
|
||||
.returning(['id', 'requeues'])
|
||||
await this.addJob(group.name, Number(row.id), 0, startId, endId, backfill)
|
||||
}
|
||||
|
||||
private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint, backfill: boolean) {
|
||||
await QueueService.nntpFetchQueue.add(
|
||||
'fetch-headers',
|
||||
{ groupName, startId: startId.toString(), endId: endId.toString(), rangeId, ...(backfill ? { backfill: true } : {}) },
|
||||
// The job id changes per attempt: BullMQ ignores an add() for an id it still knows.
|
||||
{ jobId: this.jobId(rangeId, requeues), ...(backfill ? { priority: indexerConfig.backfill.priority } : {}) }
|
||||
)
|
||||
}
|
||||
|
||||
private jobId(rangeId: number, requeues: number) {
|
||||
return `range-${rangeId}-${requeues}`
|
||||
}
|
||||
|
||||
async markDone(rangeId: number) {
|
||||
await db.from('fetch_ranges').where('id', rangeId).update({ status: 'done', error: null, updated_at: new Date() })
|
||||
}
|
||||
|
||||
async markFailed(rangeId: number, error: string) {
|
||||
await db.from('fetch_ranges').where('id', rangeId).update({ status: 'failed', error, updated_at: new Date() })
|
||||
}
|
||||
|
||||
/** Re-queues failed ranges, and queued ranges whose job no longer exists. */
|
||||
async requeue(group: Group): Promise<number> {
|
||||
const now = Date.now()
|
||||
const candidates = await db
|
||||
.from('fetch_ranges')
|
||||
.where('group_id', group.id)
|
||||
.where('requeues', '<', MAX_REQUEUES)
|
||||
.where((q) =>
|
||||
q
|
||||
.where((f) => f.where('status', 'failed').where('updated_at', '<', new Date(now - REQUEUE_AFTER_MS)))
|
||||
.orWhere((l) => l.where('status', 'queued').where('updated_at', '<', new Date(now - LOST_AFTER_MS)))
|
||||
)
|
||||
.orderBy('start_id')
|
||||
|
||||
let requeued = 0
|
||||
for (const range of candidates) {
|
||||
if (range.status === 'queued') {
|
||||
const job = await QueueService.nntpFetchQueue.getJob(this.jobId(Number(range.id), range.requeues))
|
||||
const state = job ? await job.getState() : 'missing'
|
||||
if (state !== 'missing' && state !== 'failed' && state !== 'unknown') continue
|
||||
}
|
||||
await db
|
||||
.from('fetch_ranges')
|
||||
.where('id', range.id)
|
||||
.update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() })
|
||||
await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id), Boolean(range.backfill))
|
||||
requeued++
|
||||
logger.warn(
|
||||
`Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}`
|
||||
)
|
||||
}
|
||||
return requeued
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves `groups.last_indexed_id` up to just below the first live range
|
||||
* that is not done yet (backfill ranges lie below it and don't count), and
|
||||
* forgets done ranges. Returns the new watermark.
|
||||
*/
|
||||
async advanceWatermark(group: Group): Promise<bigint> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce(
|
||||
(SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done' AND NOT backfill),
|
||||
g.last_queued_id))
|
||||
WHERE g.id = ? RETURNING last_indexed_id`,
|
||||
[group.id]
|
||||
)
|
||||
await db.from('fetch_ranges').where('group_id', group.id).where('status', 'done').delete()
|
||||
return BigInt(rows[0].last_indexed_id)
|
||||
}
|
||||
|
||||
/** Ranges that failed for good (for monitoring). */
|
||||
async abandoned(group: Group) {
|
||||
return db
|
||||
.from('fetch_ranges')
|
||||
.where('group_id', group.id)
|
||||
.where('status', 'failed')
|
||||
.where('requeues', '>=', MAX_REQUEUES)
|
||||
.count('* as count')
|
||||
.first()
|
||||
}
|
||||
}
|
||||
|
||||
export default new FetchRangeService()
|
||||
@@ -1,60 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** files ids per slice: a few seconds of reads from the HDD. */
|
||||
const SLICE = 100_000
|
||||
|
||||
/**
|
||||
* The file_keys filter (LOG.md 2026-10-06 (7)): a 64-bit hash of every
|
||||
* file_key. The insert trigger adds new keys; `fill` copies the keys of
|
||||
* files from before, in id order and a slice at a time, and marks the
|
||||
* filter complete when it reaches the files that existed when it started
|
||||
* (everything after that has been added by the trigger, which was live
|
||||
* first). FileStore uses the filter only once it is complete.
|
||||
*/
|
||||
class FileKeys {
|
||||
private completeSince = 0
|
||||
|
||||
/** Whether every file's key is in file_keys; cached once true. */
|
||||
async complete(): Promise<boolean> {
|
||||
if (this.completeSince) return true
|
||||
const row = await db.from('file_keys_fill').where('id', 1).first()
|
||||
if (row?.complete) this.completeSince = Date.now()
|
||||
return Boolean(row?.complete)
|
||||
}
|
||||
|
||||
/** Forgets the cached state (tests). */
|
||||
forget() {
|
||||
this.completeSince = 0
|
||||
}
|
||||
|
||||
/** Copies keys until `budgetMs` has passed or the fill is done; returns the files read and whether it's complete. */
|
||||
async fill(budgetMs: number): Promise<{ read: number; complete: boolean; nextId: number; untilId: number }> {
|
||||
const started = Date.now()
|
||||
let state = await db.from('file_keys_fill').where('id', 1).first()
|
||||
if (!state) return { read: 0, complete: false, nextId: 0, untilId: 0 }
|
||||
if (state.until_id === null) {
|
||||
// Everything inserted from here on goes through the trigger (live since the migration).
|
||||
const [{ max }] = await db.from('files').max('id as max')
|
||||
await db.from('file_keys_fill').where('id', 1).update({ until_id: Number(max ?? 0), updated_at: new Date() })
|
||||
state = await db.from('file_keys_fill').where('id', 1).first()
|
||||
}
|
||||
let next = Number(state.next_id)
|
||||
const until = Number(state.until_id)
|
||||
let read = 0
|
||||
while (next <= until && Date.now() - started < budgetMs) {
|
||||
const to = Math.min(until, next + SLICE - 1)
|
||||
const result = await db.rawQuery(
|
||||
`INSERT INTO file_keys (key_hash)
|
||||
SELECT DISTINCT hashtextextended(file_key, 0) FROM files WHERE id BETWEEN ? AND ? AND file_key IS NOT NULL
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[next, to]
|
||||
)
|
||||
read += Number(result.rowCount ?? 0)
|
||||
next = to + 1
|
||||
await db.from('file_keys_fill').where('id', 1).update({ next_id: next, complete: next > until, updated_at: new Date() })
|
||||
}
|
||||
return { read, complete: next > until, nextId: next, untilId: until }
|
||||
}
|
||||
}
|
||||
|
||||
export default new FileKeys()
|
||||
@@ -1,50 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/** Ids per partition (as the migration that partitioned files: …150). */
|
||||
export const FILE_PARTITION_SIZE = 4_000_000
|
||||
/**
|
||||
* Ids kept covered beyond the newest file: four partitions (16M ids, about a day of
|
||||
* seeding), so a few sweeps that can't create one are harmless.
|
||||
*/
|
||||
const AHEAD = 4 * FILE_PARTITION_SIZE
|
||||
|
||||
/**
|
||||
* `files` is partitioned by id (CONCEPT.md D39). An insert with an id no
|
||||
* partition covers fails, so gc:scheduler calls ensure() every sweep to keep
|
||||
* four partitions ready ahead of the newest id. Each new partition gets its
|
||||
* own unique index on file_key (FileStore.insert checks the older ones),
|
||||
* in the parent's tablespace (the HDD on homelab).
|
||||
*/
|
||||
class FilePartitions {
|
||||
async ensure(): Promise<string[]> {
|
||||
const created: string[] = []
|
||||
while (true) {
|
||||
// The sequence, not max(id): cheaper, and it counts ids handed out but not committed yet.
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT (SELECT last_value FROM files_id_seq) AS max_id,
|
||||
-- ''{0,1}, not ''?: knex reads ? as a binding.
|
||||
max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper
|
||||
FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = 'files'::regclass`
|
||||
)
|
||||
const maxId = Number(rows[0].max_id)
|
||||
const upper = Number(rows[0].upper)
|
||||
if (upper - maxId >= AHEAD) return created
|
||||
const name = `files_p${upper}`
|
||||
// CREATE … PARTITION OF locks files and its partitions one by one, which deadlocked with
|
||||
// file inserts (11 failed sweeps on 2026-10-07). Taking the parent's lock first, in one
|
||||
// statement, can only wait; after 10 s it gives up until the next sweep.
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`SET LOCAL lock_timeout = '10s'`)
|
||||
await trx.rawQuery(`LOCK TABLE files IN ACCESS EXCLUSIVE MODE`)
|
||||
await trx.rawQuery(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${upper}) TO (${upper + FILE_PARTITION_SIZE})`)
|
||||
await trx.rawQuery(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
|
||||
})
|
||||
logger.info(`files: partition ${name} created (ids ${upper}–${upper + FILE_PARTITION_SIZE - 1}).`)
|
||||
created.push(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new FilePartitions()
|
||||
@@ -1,138 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import FileKeys from '#services/FileKeys'
|
||||
import type { QueryClientContract } from '@adonisjs/lucid/types/database'
|
||||
import { packSegments, unpackSegments, type Segment } from '#lib/nzb/segments'
|
||||
|
||||
/**
|
||||
* The only code that knows how `files` keeps its segment lists (CONCEPT.md
|
||||
* D31): compressed in `files.segments`, with the first segment's message-id
|
||||
* in the indexed `first_message_id` for lookups. Read them with
|
||||
* `segmentsOf(file)`; write new files with `FileStore.insert`.
|
||||
*/
|
||||
|
||||
/** A new `files` row, with its segments as a list. */
|
||||
export type NewFile = Record<string, unknown> & { segments: Segment[] }
|
||||
|
||||
const unpacked = new WeakMap<object, Segment[]>()
|
||||
|
||||
/** The segments of a `files` row (a row from `db.from('files')`), in order. */
|
||||
export function segmentsOf(file: { segments?: Buffer | Segment[] | null }): Segment[] {
|
||||
const { segments } = file
|
||||
if (!segments) return []
|
||||
if (Array.isArray(segments)) return segments
|
||||
let list = unpacked.get(file)
|
||||
if (!list) {
|
||||
list = unpackSegments(segments)
|
||||
unpacked.set(file, list)
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
/** Stored form of a segment list, for the File model's column. */
|
||||
export function storedSegments(segments: Segment[]) {
|
||||
return packSegments(segments)
|
||||
}
|
||||
|
||||
export function loadedSegments(stored: Buffer | null): Segment[] {
|
||||
return stored ? unpackSegments(stored) : []
|
||||
}
|
||||
|
||||
/** What inspection learns about a file, kept beside `files` (LOG.md 2026-10-08 (14)). */
|
||||
export interface FileDetails {
|
||||
content_type?: string | null
|
||||
archive?: unknown
|
||||
}
|
||||
|
||||
const DETAIL_COLUMNS = ['content_type', 'archive'] as const
|
||||
|
||||
/** Rows per INSERT, well below Postgres' 65,535 bind parameters (tests lower it). */
|
||||
export const insertLimits = { rows: 2000 }
|
||||
|
||||
class FileStore {
|
||||
/** Sets some of a file's details, leaving the others as they are. */
|
||||
async setDetails(fileId: number, details: FileDetails, client: Pick<QueryClientContract, 'rawQuery'> = db) {
|
||||
const columns = DETAIL_COLUMNS.filter((c) => details[c] !== undefined)
|
||||
if (columns.length === 0) return
|
||||
const values = columns.map((c) => (c === 'archive' && details[c] !== null ? JSON.stringify(details[c]) : details[c]))
|
||||
await client.rawQuery(
|
||||
`INSERT INTO file_details (file_id, ${columns.join(', ')}) VALUES (?, ${columns.map(() => '?').join(', ')})
|
||||
ON CONFLICT (file_id) DO UPDATE SET ${columns.map((c) => `${c} = excluded.${c}`).join(', ')}`,
|
||||
[fileId, ...(values as any[])]
|
||||
)
|
||||
}
|
||||
|
||||
/** A file's details (empty when it has none). */
|
||||
async details(fileId: number): Promise<FileDetails> {
|
||||
return (await db.from('file_details').where('file_id', fileId).select('content_type', 'archive').first()) ?? {}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts files, skipping those whose `file_key` exists. Returns the rows
|
||||
* that were inserted (id, filename, file_key). `files` is partitioned by id
|
||||
* (CONCEPT.md D39) and each partition has its own unique file_key index:
|
||||
* once file_keys (a 64-bit hash of every key) is complete, it says which keys
|
||||
* may exist, and only those are looked up in the partitions; two
|
||||
* writers adding the same new key at once meet in the newest partition's index.
|
||||
*/
|
||||
async insert(
|
||||
files: NewFile[],
|
||||
client: Pick<QueryClientContract, 'knexQuery' | 'rawQuery'> = db
|
||||
): Promise<{ id: number; filename: string; file_key: string | null }[]> {
|
||||
if (files.length === 0) return []
|
||||
const keys = [...new Set(files.map((file) => file.file_key).filter((key): key is string => typeof key === 'string'))]
|
||||
const known = new Set<string>()
|
||||
const filtered = await FileKeys.complete()
|
||||
for (let i = 0; i < keys.length; i += 5000) {
|
||||
const chunk = keys.slice(i, i + 5000)
|
||||
if (!filtered) {
|
||||
const found = await client.knexQuery().from('files').whereIn('file_key', chunk).select('file_key')
|
||||
for (const row of found) known.add(row.file_key)
|
||||
continue
|
||||
}
|
||||
const { rows } = await client.rawQuery(
|
||||
`SELECT k.key FROM unnest(?::text[]) AS k(key)
|
||||
WHERE EXISTS (SELECT 1 FROM file_keys f WHERE f.key_hash = hashtextextended(k.key, 0))`,
|
||||
[chunk]
|
||||
)
|
||||
const maybe = rows.map((row: { key: string }) => row.key)
|
||||
if (maybe.length === 0) continue
|
||||
const found = await client.knexQuery().from('files').whereIn('file_key', maybe).select('file_key')
|
||||
for (const row of found) known.add(row.file_key)
|
||||
}
|
||||
const seen = new Set<string>()
|
||||
const rows = files
|
||||
.filter((file) => {
|
||||
const key = file.file_key as string | null | undefined
|
||||
if (!key) return true
|
||||
if (known.has(key) || seen.has(key)) return false
|
||||
seen.add(key)
|
||||
return true
|
||||
})
|
||||
.map(({ segments, ...file }) => {
|
||||
const sorted = [...segments].sort((a, b) => a.number - b.number)
|
||||
return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
|
||||
})
|
||||
if (rows.length === 0) return []
|
||||
// Details given with a new file (e.g. a known content type) go to file_details.
|
||||
const details = rows.map((row: Record<string, unknown>) => {
|
||||
const d: FileDetails = {}
|
||||
for (const c of DETAIL_COLUMNS) if (c in row) (d as any)[c] = row[c]
|
||||
for (const c of DETAIL_COLUMNS) delete row[c]
|
||||
return d
|
||||
})
|
||||
// A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
|
||||
const inserted: { id: number; filename: string; file_key: string | null }[] = []
|
||||
for (let i = 0; i < rows.length; i += insertLimits.rows) {
|
||||
const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + insertLimits.rows)).onConflict().ignore().returning(['id', 'filename', 'file_key'])
|
||||
for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename, file_key: row.file_key ?? null })
|
||||
}
|
||||
const detailsByKey = new Map(rows.map((row: Record<string, unknown>, i: number) => [row.file_key ?? row.filename, details[i]]))
|
||||
for (const row of inserted) {
|
||||
const d = detailsByKey.get(row.file_key ?? row.filename)
|
||||
if (d && Object.keys(d).length > 0) await this.setDetails(row.id, d, client)
|
||||
}
|
||||
return inserted
|
||||
}
|
||||
}
|
||||
|
||||
export default new FileStore()
|
||||
@@ -1,217 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
import { explored, signatureGroup, signatureKey, uploadTool } from '#lib/garbage/index'
|
||||
|
||||
/** What the filter needs to know about an upload before it is stored. */
|
||||
export interface Upload {
|
||||
/** The posted file name. */
|
||||
name: string
|
||||
/** Any of its articles' Message-IDs (one tool posts all of a file). */
|
||||
messageId: string | null | undefined
|
||||
groups: unknown
|
||||
/** Its release's key (the file key when it has none): kept or dropped as a whole. */
|
||||
releaseKey: string
|
||||
}
|
||||
|
||||
/** Contents that can't name a random-named release without a key (LOG.md 2026-10-08 (5)). */
|
||||
const CONTAINERS = ['rclone', 'nzbe', 'yencbin']
|
||||
|
||||
/** How long the dropped signatures are cached per process. */
|
||||
const CACHE_MS = 5 * 60 * 1000
|
||||
|
||||
export function isRandomName(name: string) {
|
||||
return looksRandom(fileStem(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* Garbage: random-named uploads that never get a real name and have no
|
||||
* known password (CONCEPT.md D42). Two filters:
|
||||
* - before storing (seeding, ingest): random-named uploads whose signature
|
||||
* (posting tool, first group) is dropped, or whose release was dropped
|
||||
* after its probe, aren't stored;
|
||||
* - after a probe: a random-named release whose probes are all done, that
|
||||
* is still without a real name, password, names from its files or spot,
|
||||
* and is encrypted, a container or missing its first article, is deleted.
|
||||
* The dropped signatures are learned from a sample of probed releases
|
||||
* (learn(), gc:scheduler).
|
||||
*/
|
||||
class GarbageFilter {
|
||||
private dropped: Set<string> | null = null
|
||||
private loadedAt = 0
|
||||
|
||||
/** The dropped signatures, cached for CACHE_MS. */
|
||||
async droppedSignatures(): Promise<Set<string>> {
|
||||
if (this.dropped && Date.now() - this.loadedAt < CACHE_MS) return this.dropped
|
||||
const rows = await db.from('upload_signatures').where('dropped', true).select('tool', 'group_name')
|
||||
this.dropped = new Set(rows.map((row) => signatureKey(row.tool, row.group_name)))
|
||||
this.loadedAt = Date.now()
|
||||
return this.dropped
|
||||
}
|
||||
|
||||
/** Forgets the cached signatures (after learn(), and in tests). */
|
||||
reset() {
|
||||
this.dropped = null
|
||||
}
|
||||
|
||||
/** Whether a random-named upload's signature is dropped and it isn't in the explored share. */
|
||||
private droppedBySignature(upload: Upload, signatures: Set<string>) {
|
||||
if (signatures.size === 0) return false
|
||||
if (!signatures.has(signatureKey(uploadTool(upload.messageId), signatureGroup(upload.groups)))) return false
|
||||
return !explored(upload.releaseKey, indexerConfig.garbage.explorePercent)
|
||||
}
|
||||
|
||||
/**
|
||||
* The items worth storing: drops random-named uploads of dropped signatures
|
||||
* (but explorePercent of their releases) and of dropped releases.
|
||||
*/
|
||||
async keep<T>(items: T[], upload: (item: T) => Upload): Promise<{ kept: T[]; dropped: number }> {
|
||||
if (!indexerConfig.garbage.enabled || items.length === 0) return { kept: items, dropped: 0 }
|
||||
const signatures = await this.droppedSignatures()
|
||||
// Per release, kept as a whole when any of its files has a readable name. A file is
|
||||
// looked at once: an ingest batch has many articles of each.
|
||||
const verdicts = new Map<string, 'keep' | 'drop' | 'check'>()
|
||||
const seen = new Set<string>()
|
||||
for (const item of items) {
|
||||
const u = upload(item)
|
||||
if (verdicts.get(u.releaseKey) === 'keep' || seen.has(`${u.releaseKey}\n${u.name}`)) continue
|
||||
seen.add(`${u.releaseKey}\n${u.name}`)
|
||||
if (!isRandomName(u.name)) verdicts.set(u.releaseKey, 'keep')
|
||||
else if (!verdicts.has(u.releaseKey)) verdicts.set(u.releaseKey, this.droppedBySignature(u, signatures) ? 'drop' : 'check')
|
||||
}
|
||||
const check = [...verdicts].filter(([, v]) => v === 'check').map(([key]) => key)
|
||||
for (let i = 0; i < check.length; i += 5000) {
|
||||
const rows = await db.from('dropped_releases').whereIn('release_key', check.slice(i, i + 5000)).select('release_key')
|
||||
for (const row of rows) verdicts.set(row.release_key, 'drop')
|
||||
}
|
||||
const kept = items.filter((item) => verdicts.get(upload(item).releaseKey) !== 'drop')
|
||||
return { kept, dropped: items.length - kept.length }
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a stored file's probe can be skipped: a random-named file of a
|
||||
* dropped signature, outside the explored share (files stored before its
|
||||
* signature was dropped). Its binary stops waiting for inspection.
|
||||
*/
|
||||
async skipProbe(file: { id: number | string; filename: string; real_name?: string | null; first_message_id?: string | null; groups: unknown; release_id?: number | string | null }) {
|
||||
if (!indexerConfig.garbage.enabled || file.real_name || !isRandomName(file.filename)) return false
|
||||
const signatures = await this.droppedSignatures()
|
||||
if (!signatures.has(signatureKey(uploadTool(file.first_message_id), signatureGroup(file.groups)))) return false
|
||||
const release = file.release_id ? await db.from('releases').where('id', file.release_id).select('release_key').first() : null
|
||||
if (explored(release?.release_key ?? String(file.id), indexerConfig.garbage.explorePercent)) return false
|
||||
await db.from('binaries').where('file_id', file.id).where('awaiting_inspection', true).update({ awaiting_inspection: null })
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* After a file's probe has been applied: notes its release for learning
|
||||
* (a sample) and deletes the release when it is garbage. Never throws:
|
||||
* the probe's result is stored already. Returns why the release was
|
||||
* deleted, or null.
|
||||
*/
|
||||
async afterProbe(fileId: number, contentType: string | null): Promise<string | null> {
|
||||
try {
|
||||
const file = await db.from('files').where('id', fileId).select('id', 'filename', 'real_name', 'first_message_id', 'groups', 'release_id').first()
|
||||
if (!file?.release_id || !isRandomName(String(file.filename))) return null
|
||||
const releaseId = Number(file.release_id)
|
||||
if (releaseId % indexerConfig.garbage.probeSample === 0) {
|
||||
await db.rawQuery(`INSERT INTO signature_probes (release_id, tool, group_name) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [
|
||||
releaseId,
|
||||
uploadTool(file.first_message_id),
|
||||
signatureGroup(file.groups),
|
||||
])
|
||||
}
|
||||
if (!indexerConfig.garbage.enabled) return null
|
||||
return await this.dropIfGarbage(releaseId, contentType)
|
||||
} catch (error: any) {
|
||||
logger.warn(`Garbage check after probing file ${fileId}: ${error.message}`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private async dropIfGarbage(releaseId: number, contentType: string | null): Promise<string | null> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT r.release_key, r.container, r.encrypted,
|
||||
is_real_release_name(r.name) OR r.password IS NOT NULL OR coalesce(r.search_files, '') <> ''
|
||||
OR EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id) AS named,
|
||||
EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id AND b.awaiting_inspection) AS waiting
|
||||
FROM releases r WHERE r.id = ?`,
|
||||
[releaseId]
|
||||
)
|
||||
const release = rows[0]
|
||||
if (!release || release.named || release.waiting) return null
|
||||
const reason = CONTAINERS.includes(release.container) ? release.container : release.encrypted === true ? 'encrypted' : contentType === 'missing' ? 'missing' : null
|
||||
if (!reason) return null
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`INSERT INTO dropped_releases (release_key, reason) VALUES (?, ?) ON CONFLICT DO NOTHING`, [release.release_key, reason])
|
||||
await trx.rawQuery(`DELETE FROM binaries WHERE release_id = ?`, [releaseId])
|
||||
await trx.rawQuery(`DELETE FROM files WHERE release_id = ?`, [releaseId])
|
||||
await trx.rawQuery(`DELETE FROM releases WHERE id = ?`, [releaseId])
|
||||
})
|
||||
return reason
|
||||
}
|
||||
|
||||
/**
|
||||
* Learns the signatures from the probed releases of the last windowDays:
|
||||
* dropped when at least minProbes are known and at most maxNamedRate got a
|
||||
* real name, password, names from their files or a spot. A signature with
|
||||
* fewer probes keeps its verdict (a dropped one only gets explored probes).
|
||||
* Releases deleted as garbage count as unnamed. Returns the signatures
|
||||
* whose verdict changed.
|
||||
*/
|
||||
async learn(): Promise<{ signatures: number; dropped: number; changed: string[] }> {
|
||||
const { minProbes, maxNamedRate, windowDays } = indexerConfig.garbage
|
||||
await db.rawQuery(`DELETE FROM signature_probes WHERE probed_at < now() - make_interval(days => ?)`, [windowDays])
|
||||
const before = new Map((await db.from('upload_signatures').select('tool', 'group_name', 'dropped')).map((row) => [signatureKey(row.tool, row.group_name), row.dropped]))
|
||||
await db.rawQuery(
|
||||
`INSERT INTO upload_signatures (tool, group_name, probed, named, dropped, updated_at)
|
||||
SELECT tool, group_name, probed, named, probed >= ?::int AND named <= probed * ?::float8, now()
|
||||
FROM (
|
||||
SELECT p.tool, p.group_name, count(*)::int AS probed,
|
||||
(count(*) FILTER (WHERE is_real_release_name(r.name) OR r.password IS NOT NULL OR coalesce(r.search_files, '') <> ''
|
||||
OR EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id)))::int AS named
|
||||
FROM signature_probes p LEFT JOIN releases r ON r.id = p.release_id
|
||||
GROUP BY p.tool, p.group_name
|
||||
) s
|
||||
ON CONFLICT (tool, group_name) DO UPDATE SET
|
||||
probed = excluded.probed, named = excluded.named, updated_at = now(),
|
||||
dropped = CASE WHEN excluded.probed >= ?::int THEN excluded.dropped ELSE upload_signatures.dropped END`,
|
||||
[minProbes, maxNamedRate, minProbes]
|
||||
)
|
||||
const after = await db.from('upload_signatures').select('tool', 'group_name', 'dropped', 'probed', 'named')
|
||||
const changed = after
|
||||
.filter((row) => (before.get(signatureKey(row.tool, row.group_name)) ?? false) !== row.dropped)
|
||||
.map((row) => `${row.tool} ${row.group_name || '(no group)'}: ${row.dropped ? 'dropped' : 'kept'} (${row.named}/${row.probed} named)`)
|
||||
this.reset()
|
||||
return { signatures: after.length, dropped: after.filter((row) => row.dropped).length, changed }
|
||||
}
|
||||
|
||||
/**
|
||||
* Fills signature_probes from a sample of files probed before it existed
|
||||
* (garbage:learn --seed): random-named files with a content type, from
|
||||
* TABLESAMPLE SYSTEM (percent) of files. Returns the releases added.
|
||||
*/
|
||||
async seed(percent: number): Promise<number> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT ON (f.release_id) f.release_id, f.first_message_id, f.groups
|
||||
FROM files f TABLESAMPLE SYSTEM (?) JOIN file_details d ON d.file_id = f.id
|
||||
WHERE d.content_type IS NOT NULL AND f.release_id IS NOT NULL AND NOT is_readable_name(f.filename)`,
|
||||
[percent]
|
||||
)
|
||||
let added = 0
|
||||
for (let i = 0; i < rows.length; i += 2000) {
|
||||
const chunk = rows.slice(i, i + 2000)
|
||||
const result = await db.rawQuery(
|
||||
`INSERT INTO signature_probes (release_id, tool, group_name)
|
||||
SELECT * FROM unnest(?::bigint[], ?::text[], ?::text[]) ON CONFLICT DO NOTHING`,
|
||||
[chunk.map((row: any) => row.release_id), chunk.map((row: any) => uploadTool(row.first_message_id)), chunk.map((row: any) => signatureGroup(row.groups))]
|
||||
)
|
||||
added += result.rowCount ?? 0
|
||||
}
|
||||
return added
|
||||
}
|
||||
}
|
||||
|
||||
export default new GarbageFilter()
|
||||
@@ -1,85 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/** Only indexes at least this big are worth checking. */
|
||||
const MIN_BYTES = 50 * 1024 * 1024
|
||||
/** Leaf pages less full than this (percent) mean the index is mostly empty space. */
|
||||
const MAX_LEAF_DENSITY = 50
|
||||
|
||||
export interface Rebuilt {
|
||||
index: string
|
||||
density: number
|
||||
before: number
|
||||
after: number
|
||||
}
|
||||
|
||||
/**
|
||||
* GC deletes millions of rows (single-article binaries, copied parts, empty
|
||||
* releases). Vacuum makes their index entries reusable but doesn't compact
|
||||
* the index, so B-trees on the churning tables end up 10–20% full (LOG.md
|
||||
* 2026-10-01). Rebuilding them online brings them back to size (CONCEPT.md
|
||||
* D33). Needs the pgstattuple extension.
|
||||
*/
|
||||
class IndexMaintenance {
|
||||
async rebuildBloated({ dryRun = false } = {}): Promise<Rebuilt[]> {
|
||||
const { rows: extension } = await db.rawQuery(`SELECT 1 FROM pg_extension WHERE extname = 'pgstattuple'`)
|
||||
if (extension.length === 0) {
|
||||
logger.warn('Index maintenance skipped: the pgstattuple extension is not installed.')
|
||||
return []
|
||||
}
|
||||
if (!dryRun) await this.dropLeftovers()
|
||||
|
||||
const { rows: candidates } = await db.rawQuery(
|
||||
`SELECT c.relname AS index, pg_relation_size(c.oid) AS bytes
|
||||
FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_am am ON am.oid = c.relam
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE am.amname = 'btree' AND n.nspname = 'public' AND i.indisvalid AND pg_relation_size(c.oid) >= ?
|
||||
ORDER BY 2 DESC`,
|
||||
[MIN_BYTES]
|
||||
)
|
||||
const rebuilt: Rebuilt[] = []
|
||||
for (const { index, bytes } of candidates) {
|
||||
const { rows } = await db.rawQuery(`SELECT avg_leaf_density FROM pgstatindex(?)`, [index])
|
||||
const density = Number(rows[0].avg_leaf_density)
|
||||
if (density >= MAX_LEAF_DENSITY) continue
|
||||
if (!dryRun && !(await this.reindex(index))) continue
|
||||
const { rows: after } = await db.rawQuery(`SELECT pg_relation_size(?::regclass) AS bytes`, [index])
|
||||
const result = { index, density, before: Number(bytes), after: Number(after[0].bytes) }
|
||||
rebuilt.push(result)
|
||||
logger.info(`Index ${index}: ${density.toFixed(0)}% full, ${mb(result.before)} → ${mb(result.after)}${dryRun ? ' (dry run)' : ''}.`)
|
||||
}
|
||||
return rebuilt
|
||||
}
|
||||
|
||||
/**
|
||||
* REINDEX CONCURRENTLY can lose a deadlock against busy writers (seen on
|
||||
* binaries during a backfill). Its half-built copy is dropped and it is
|
||||
* tried once more; a second failure skips the index until the next run.
|
||||
*/
|
||||
/** Invalid copies an interrupted REINDEX CONCURRENTLY left behind (`<index>_ccnew`, `_ccnew1`, …). */
|
||||
private async dropLeftovers(index = '') {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE NOT i.indisvalid AND c.relname LIKE ? || '%\\_ccnew%'`,
|
||||
[index]
|
||||
)
|
||||
for (const { relname } of rows) await db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS "${relname}"`)
|
||||
}
|
||||
|
||||
private async reindex(index: string): Promise<boolean> {
|
||||
for (let attempt = 1; attempt <= 2; attempt++) {
|
||||
try {
|
||||
await db.rawQuery(`REINDEX INDEX CONCURRENTLY "${index}"`)
|
||||
return true
|
||||
} catch (error: any) {
|
||||
logger.warn(`Rebuilding index ${index} failed (attempt ${attempt}): ${error.message}`)
|
||||
await this.dropLeftovers(index)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
const mb = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`
|
||||
|
||||
export default new IndexMaintenance()
|
||||
@@ -1,424 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import type { QueryClientContract } from '@adonisjs/lucid/types/database'
|
||||
import type { Knex } from 'knex'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NameService from '#services/NameService'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
import type { IngestSegment } from '#lib/ingest/segment'
|
||||
|
||||
// Header parsing is pure and lives in lib, so seeding nodes need no database (CONCEPT.md D37).
|
||||
export { segmentFromHeader, segmentFromYenc, type IngestSegment, type OverviewHeader } from '#lib/ingest/segment'
|
||||
|
||||
export interface CompletedBinary {
|
||||
id: number
|
||||
fileKey: string
|
||||
}
|
||||
|
||||
const BINARY_CHUNK = 1000
|
||||
/** last_seen_at is only rewritten when it is older than this (it drives GC at 48 h). */
|
||||
const TOUCH_MS = 10 * 60 * 1000
|
||||
|
||||
type Merge = 'least' | 'coalesce' | 'touch'
|
||||
|
||||
/**
|
||||
* Inserts rows keyed by `key` and returns their ids. Existing rows are not
|
||||
* locked unless something in them changes (`least`: an earlier value,
|
||||
* `coalesce`: a value where there was none, `touch`: a timestamp more than
|
||||
* TOUCH_MS newer). Parallel fetch jobs see mostly the same rows, and
|
||||
* `INSERT … ON CONFLICT DO UPDATE` made them wait on each other for every one.
|
||||
*/
|
||||
async function upsertKeyed(
|
||||
knex: Knex,
|
||||
table: 'binaries' | 'releases',
|
||||
key: string,
|
||||
rows: Record<string, unknown>[],
|
||||
merge: Record<string, [Merge, string]>
|
||||
): Promise<Map<string, number>> {
|
||||
const ids = new Map<string, number>()
|
||||
if (rows.length === 0) return ids
|
||||
const inserted: Record<string, any>[] = await knex(table).insert(rows).onConflict(key).ignore().returning(['id', key])
|
||||
for (const row of inserted) ids.set(row[key], Number(row.id))
|
||||
|
||||
const incoming = new Map(rows.filter((r) => !ids.has(r[key] as string)).map((r) => [r[key] as string, r]))
|
||||
if (incoming.size === 0) return ids
|
||||
const columns = Object.keys(merge)
|
||||
const existing: Record<string, any>[] = await knex(table).whereIn(key, [...incoming.keys()]).select('id', key, ...columns)
|
||||
|
||||
const changes: Record<string, unknown>[] = []
|
||||
for (const row of existing) {
|
||||
ids.set(row[key], Number(row.id))
|
||||
const next = incoming.get(row[key])!
|
||||
const changed = columns.some((column) => {
|
||||
const [kind] = merge[column]
|
||||
const current = row[column]
|
||||
const value = next[column]
|
||||
if (value === null || value === undefined) return false
|
||||
if (kind === 'coalesce') return current === null
|
||||
if (kind === 'least') return current === null || value < current
|
||||
return current === null || +(value as Date) - +current > TOUCH_MS
|
||||
})
|
||||
if (changed) changes.push({ id: Number(row.id), ...Object.fromEntries(columns.map((c) => [c, next[c] ?? null])) })
|
||||
}
|
||||
if (changes.length === 0) return ids
|
||||
|
||||
const set = columns
|
||||
.map((c) => {
|
||||
const [kind] = merge[c]
|
||||
const fn = kind === 'least' ? 'least' : kind === 'touch' ? 'greatest' : 'coalesce'
|
||||
return `${c} = ${fn}(t.${c}, v.${c})`
|
||||
})
|
||||
.join(', ')
|
||||
const types = columns.map((c) => `${c} ${merge[c][1]}`).join(', ')
|
||||
changes.sort((a, b) => (a.id as number) - (b.id as number))
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
await knex.raw(
|
||||
`UPDATE ${table} t SET ${set}
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, ${types})
|
||||
WHERE t.id = v.id`,
|
||||
[JSON.stringify(changes)]
|
||||
)
|
||||
return ids
|
||||
} catch (error: any) {
|
||||
// Rare now, but two jobs can still update overlapping rows in a different order.
|
||||
if (error.code !== '40P01' || attempt >= 3) throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
/** Completed binaries per worker:file job. */
|
||||
const FILE_BATCH = 100
|
||||
|
||||
/** Size-only matching below this is too ambiguous (nfo, sfv, small volumes). */
|
||||
export const PAR2_SIZE_ONLY_MIN = 1024 * 1024
|
||||
/** A par2 set and its files are posted together; don't match across days. */
|
||||
export const PAR2_MATCH_WINDOW_SECONDS = 12 * 60 * 60
|
||||
const PART_CHUNK = 2000
|
||||
|
||||
function chunks<T>(items: T[], size: number): T[][] {
|
||||
const result: T[][] = []
|
||||
for (let i = 0; i < items.length; i += size) result.push(items.slice(i, i + size))
|
||||
return result
|
||||
}
|
||||
|
||||
function validDate(date?: Date) {
|
||||
return date && !Number.isNaN(date.getTime()) ? date : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores segments in `binaries`/`parts` and queues every binary that became
|
||||
* complete exactly once. Idempotent: ingesting the same segments again is a
|
||||
* no-op. See CONCEPT.md (D12).
|
||||
*/
|
||||
class IngestService {
|
||||
async ingest(batch: IngestSegment[]): Promise<CompletedBinary[]> {
|
||||
// Garbage isn't stored at all: no release, binary or parts (CONCEPT.md D42).
|
||||
const { kept: segments } = await GarbageFilter.keep(batch, (s) => ({
|
||||
name: s.name,
|
||||
messageId: s.messageId,
|
||||
groups: s.groups,
|
||||
releaseKey: s.release?.key ?? s.fileKey,
|
||||
}))
|
||||
if (segments.length === 0) return []
|
||||
|
||||
// Short statements, no enclosing transaction: every step is idempotent, and
|
||||
// row locks held for a whole batch made parallel fetch jobs wait on each other.
|
||||
const binaryIds = await (async () => {
|
||||
const knex = db.connection().getWriteClient()
|
||||
const now = new Date()
|
||||
const releaseIds = await this.upsertReleases(knex, segments, now)
|
||||
|
||||
// One row per file_key (ON CONFLICT cannot touch a row twice per statement),
|
||||
// sorted so concurrent workers lock rows in the same order and cannot deadlock.
|
||||
const binaries = new Map<string, Record<string, unknown>>()
|
||||
for (const segment of segments) {
|
||||
const postedAt = validDate(segment.postedAt)
|
||||
const existing = binaries.get(segment.fileKey)
|
||||
if (existing) {
|
||||
if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
|
||||
existing.posted_at = postedAt
|
||||
}
|
||||
continue
|
||||
}
|
||||
binaries.set(segment.fileKey, {
|
||||
file_key: segment.fileKey,
|
||||
poster: segment.poster,
|
||||
subject: segment.subject,
|
||||
name: segment.name,
|
||||
total_parts: segment.totalParts,
|
||||
groups: JSON.stringify(segment.groups),
|
||||
posted_at: postedAt,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
release_id: segment.release ? releaseIds.get(segment.release.key) : null,
|
||||
size: segment.size ?? null,
|
||||
md5_16k: segment.md5_16k ?? null,
|
||||
})
|
||||
}
|
||||
const binaryRows = [...binaries.values()].sort((a, b) =>
|
||||
(a.file_key as string).localeCompare(b.file_key as string)
|
||||
)
|
||||
|
||||
const idsByKey = new Map<string, number>()
|
||||
for (const chunk of chunks(binaryRows, BINARY_CHUNK)) {
|
||||
const ids = await upsertKeyed(knex, 'binaries', 'file_key', chunk, {
|
||||
last_seen_at: ['touch', 'timestamptz'],
|
||||
posted_at: ['least', 'timestamptz'],
|
||||
release_id: ['coalesce', 'bigint'],
|
||||
md5_16k: ['coalesce', 'varchar'],
|
||||
})
|
||||
for (const [key, id] of ids) idsByKey.set(key, id)
|
||||
}
|
||||
|
||||
// A binary that is a file already has its segment list in `files`; its
|
||||
// header parts were dropped then. Crossposts in other groups, reposts
|
||||
// and backfill ranges keep bringing the same articles, and inserting
|
||||
// them again grew `parts` to 56M rows nobody reads (LOG.md 2026-10-04).
|
||||
const done = new Set<number>()
|
||||
const ids = [...idsByKey.values()]
|
||||
for (const chunk of chunks(ids, BINARY_CHUNK)) {
|
||||
const { rows } = await knex.raw('SELECT id FROM binaries WHERE id = ANY(?) AND file_id IS NOT NULL', [chunk])
|
||||
for (const row of rows) done.add(Number(row.id))
|
||||
}
|
||||
|
||||
const partRows = segments
|
||||
.map((segment) => ({
|
||||
binary_id: idsByKey.get(segment.fileKey)!,
|
||||
number: segment.number,
|
||||
message_id: segment.messageId,
|
||||
bytes: segment.bytes,
|
||||
article_number: segment.articleNumber?.toString() ?? null,
|
||||
pcrc32: segment.pcrc32 ?? null,
|
||||
part_size: segment.partSize ?? null,
|
||||
file_crc32: segment.fileCrc32 ?? null,
|
||||
md5_16k: segment.md5_16k ?? null,
|
||||
}))
|
||||
const weakRows = partRows.filter((row, i) => segments[i].weakIdentity && !(done.has(row.binary_id) && row.pcrc32 === null))
|
||||
// Body-derived parts (with a per-part CRC) are kept, as worker:file does.
|
||||
const newParts = partRows.filter((row) => !(done.has(row.binary_id) && row.pcrc32 === null))
|
||||
|
||||
newParts.sort((a, b) => a.binary_id - b.binary_id || a.number - b.number)
|
||||
// Parts and the binaries' stored-part counts in one statement, so the count can't miss a
|
||||
// part (LOG.md 2026-10-06 (7)); claimCompleted compares it instead of counting parts. NULL
|
||||
// (binaries from before part_count) is counted once: the statement's own rows aren't in
|
||||
// its snapshot, so they're added on top. Concurrent batches can lock binaries in a
|
||||
// different order: a deadlock rolls the whole statement back, and it is retried.
|
||||
for (const chunk of chunks(newParts, PART_CHUNK)) {
|
||||
const insert = knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore().returning('binary_id').toSQL()
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
await knex.raw(
|
||||
`WITH added AS (${insert.sql}),
|
||||
counted AS (SELECT binary_id, count(*)::int AS n FROM added GROUP BY binary_id)
|
||||
UPDATE binaries b SET part_count = CASE
|
||||
WHEN b.part_count IS NULL THEN (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) + c.n
|
||||
ELSE b.part_count + c.n END
|
||||
FROM counted c WHERE b.id = c.binary_id`,
|
||||
insert.bindings as any[]
|
||||
)
|
||||
break
|
||||
} catch (error: any) {
|
||||
if (error.code !== '40P01' || attempt >= 5) throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (weakRows.length > 0) await this.flagConflicts(db.connection(), weakRows)
|
||||
|
||||
return [...idsByKey.values()]
|
||||
})()
|
||||
|
||||
// par2 naming needs a size, which only body-derived segments have; for
|
||||
// header batches these queries were most of Postgres' work.
|
||||
const sized = [...new Set(segments.filter((s) => s.size != null).map((s) => s.fileKey))]
|
||||
if (sized.length > 0) {
|
||||
const sizedIds = (await db.from('binaries').whereIn('file_key', sized).select('id')).map((r) => Number(r.id))
|
||||
await this.matchPar2Releases(sizedIds)
|
||||
await NameService.resolve(sizedIds)
|
||||
}
|
||||
|
||||
// After commit, so the last writer of a binary always sees all of its parts.
|
||||
const completed = await this.claimCompleted(binaryIds)
|
||||
await this.enqueue(completed)
|
||||
return completed
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates or touches the releases referenced by `segments` and returns
|
||||
* their ids by release key. Also used by `release:rebuild`.
|
||||
*/
|
||||
async upsertReleases(
|
||||
knex: Knex,
|
||||
segments: Pick<IngestSegment, 'release' | 'poster' | 'groups' | 'postedAt' | 'weakIdentity'>[],
|
||||
now = new Date()
|
||||
): Promise<Map<string, number>> {
|
||||
const releases = new Map<string, Record<string, unknown>>()
|
||||
for (const segment of segments) {
|
||||
const release = segment.release
|
||||
if (!release) continue
|
||||
const postedAt = validDate(segment.postedAt)
|
||||
const existing = releases.get(release.key)
|
||||
if (existing) {
|
||||
if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
|
||||
existing.posted_at = postedAt
|
||||
}
|
||||
// Not every file carries the [i/M] counter (e.g. a par2 index): take it from any that does.
|
||||
if (existing.file_total === null && release.fileTotal) existing.file_total = release.fileTotal
|
||||
continue
|
||||
}
|
||||
releases.set(release.key, {
|
||||
release_key: release.key,
|
||||
name: release.name,
|
||||
// Obfuscating posters randomise the poster, so it says nothing about the release.
|
||||
poster: segment.weakIdentity ? null : segment.poster,
|
||||
file_total: release.fileTotal ?? null,
|
||||
source: release.source,
|
||||
groups: JSON.stringify(segment.groups),
|
||||
posted_at: postedAt,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
})
|
||||
}
|
||||
|
||||
const ids = new Map<string, number>()
|
||||
const rows = [...releases.values()].sort((a, b) =>
|
||||
(a.release_key as string).localeCompare(b.release_key as string)
|
||||
)
|
||||
for (const chunk of chunks(rows, BINARY_CHUNK)) {
|
||||
const result = await upsertKeyed(knex, 'releases', 'release_key', chunk, {
|
||||
last_seen_at: ['touch', 'timestamptz'],
|
||||
posted_at: ['least', 'timestamptz'],
|
||||
file_total: ['coalesce', 'integer'],
|
||||
source: ['coalesce', 'varchar'],
|
||||
})
|
||||
for (const [key, id] of result) ids.set(key, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
/**
|
||||
* Assigns body-derived binaries without a release and without a
|
||||
* fingerprint to a known par2 set by size: only when that size is listed
|
||||
* once in the set, the file is large, and exactly one set nearby lists it.
|
||||
* Binaries with a fingerprint are matched exactly by NameService instead.
|
||||
*/
|
||||
async matchPar2Releases(binaryIds: number[]) {
|
||||
if (binaryIds.length === 0) return
|
||||
// Listed-once first (index-only on release_files (release_id, size): most candidate
|
||||
// sizes are a set's equal-sized volumes), the release's date only for what's left;
|
||||
// OFFSET 0 keeps that order. Was: every size match's date and count, 18 s per batch.
|
||||
await db.rawQuery(
|
||||
`UPDATE binaries b SET release_id = m.release_id
|
||||
FROM (
|
||||
SELECT c.id, min(c.release_id) AS release_id
|
||||
FROM (
|
||||
SELECT b2.id, b2.posted_at, rf.release_id
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.size = b2.size
|
||||
WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM release_files x
|
||||
WHERE x.release_id = rf.release_id AND x.size = rf.size AND x.par2_file_id <> rf.par2_file_id)
|
||||
OFFSET 0
|
||||
) c
|
||||
JOIN releases r ON r.id = c.release_id
|
||||
WHERE abs(extract(epoch FROM (r.posted_at - c.posted_at))) < ?
|
||||
GROUP BY c.id
|
||||
HAVING count(DISTINCT c.release_id) = 1
|
||||
) m
|
||||
WHERE b.id = m.id`,
|
||||
[binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS]
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Flags binaries where a stored part has a different message-id and a
|
||||
* different part CRC than the incoming one: two files were merged under
|
||||
* one identity. Identical reposts (same CRC) are ignored.
|
||||
*/
|
||||
private async flagConflicts(
|
||||
trx: QueryClientContract,
|
||||
rows: {
|
||||
binary_id: number
|
||||
number: number
|
||||
message_id: string
|
||||
bytes: number
|
||||
article_number: string | null
|
||||
pcrc32: string | null
|
||||
part_size: number | null
|
||||
file_crc32: string | null
|
||||
md5_16k: string | null
|
||||
}[]
|
||||
) {
|
||||
const bindings = [
|
||||
rows.map((r) => r.binary_id),
|
||||
rows.map((r) => r.number),
|
||||
rows.map((r) => r.message_id),
|
||||
rows.map((r) => r.bytes),
|
||||
rows.map((r) => r.article_number),
|
||||
rows.map((r) => r.pcrc32),
|
||||
rows.map((r) => r.part_size),
|
||||
rows.map((r) => r.file_crc32),
|
||||
rows.map((r) => r.md5_16k),
|
||||
]
|
||||
// Keep the extra articles: SplitService separates the volumes from them.
|
||||
const conflicts = await trx.rawQuery(
|
||||
`INSERT INTO part_conflicts (binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
|
||||
SELECT v.binary_id, v.number, v.message_id, v.bytes, v.article_number, v.pcrc32, v.part_size, v.file_crc32, v.md5_16k
|
||||
FROM unnest(?::bigint[], ?::int[], ?::text[], ?::int[], ?::bigint[], ?::text[], ?::int[], ?::text[], ?::text[])
|
||||
AS v(binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
|
||||
JOIN parts p ON p.binary_id = v.binary_id AND p.number = v.number
|
||||
WHERE p.message_id <> v.message_id
|
||||
-- Same part CRC: the same data posted again (a repost), not another file.
|
||||
AND (p.pcrc32 IS NULL OR v.pcrc32 IS NULL OR p.pcrc32 <> v.pcrc32)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING binary_id`,
|
||||
bindings as any
|
||||
)
|
||||
const binaryIds: number[] = [
|
||||
...new Set<number>(conflicts.rows.map((r: { binary_id: string }) => Number(r.binary_id))),
|
||||
]
|
||||
if (binaryIds.length === 0) return
|
||||
const result = await trx.rawQuery(
|
||||
`UPDATE binaries SET conflicted_at = now()
|
||||
WHERE conflicted_at IS NULL AND id = ANY(?) RETURNING name`,
|
||||
[binaryIds]
|
||||
)
|
||||
for (const row of result.rows) {
|
||||
logger.warn(`Binary "${row.name}" has conflicting parts; it will be split into volumes.`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks complete binaries among `binaryIds` as claimed. The single UPDATE
|
||||
* locks each row, so concurrent callers never claim the same binary twice.
|
||||
*/
|
||||
async claimCompleted(binaryIds: number[]): Promise<CompletedBinary[]> {
|
||||
if (binaryIds.length === 0) return []
|
||||
const result = await db.rawQuery(
|
||||
`UPDATE binaries b SET completed_at = now()
|
||||
WHERE b.id = ANY(?) AND b.completed_at IS NULL AND b.conflicted_at IS NULL
|
||||
AND coalesce(b.part_count, (SELECT count(*) FROM parts p WHERE p.binary_id = b.id)) >= b.total_parts
|
||||
RETURNING b.id, b.file_key`,
|
||||
[binaryIds]
|
||||
)
|
||||
return result.rows.map((row: { id: string; file_key: string }) => ({
|
||||
id: Number(row.id),
|
||||
fileKey: row.file_key,
|
||||
}))
|
||||
}
|
||||
|
||||
async enqueue(completed: CompletedBinary[]) {
|
||||
if (completed.length === 0) return
|
||||
// A job per FILE_BATCH binaries: worker:file stores them with a few set-based queries (LOG.md 2026-10-06 (7)).
|
||||
// jobId from the first file_key: BullMQ ignores an add() for an id that already exists.
|
||||
const jobs = []
|
||||
for (let i = 0; i < completed.length; i += FILE_BATCH) {
|
||||
const batch = completed.slice(i, i + FILE_BATCH)
|
||||
jobs.push({ name: 'process-files', data: { binaryIds: batch.map((b) => b.id) }, opts: { jobId: `${batch[0].fileKey}-${batch.length}` } })
|
||||
}
|
||||
await QueueService.fileQueue.addBulk(jobs)
|
||||
}
|
||||
}
|
||||
|
||||
export default new IngestService()
|
||||
@@ -1,66 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import env from '#start/env'
|
||||
import QueueService from '#services/QueueService'
|
||||
|
||||
/** Files per transaction when moving from the table to BullMQ. */
|
||||
const FEED_BATCH = 5000
|
||||
|
||||
/** Priorities: lower goes first. */
|
||||
export const INSPECTION_PRIORITY = { targeted: -1, normal: 0, low: 10 } as const
|
||||
|
||||
/**
|
||||
* The content inspection backlog (CONCEPT.md D21, D41; LOG.md 2026-10-07 (8)).
|
||||
* Producers add file ids to the inspection_queue table; feed() moves them
|
||||
* into BullMQ's collection-queue while fewer than INSPECTION_FEED_MAX jobs
|
||||
* wait there, where worker:collection and inspecting nodes take them as
|
||||
* before. Redis holds a working set instead of a backlog of days.
|
||||
*/
|
||||
class InspectionQueue {
|
||||
readonly feedMax = env.get('INSPECTION_FEED_MAX', 50_000)
|
||||
|
||||
/** Queues files for inspection; a file already queued keeps the higher (lower number) priority. */
|
||||
async add(fileIds: number[], priority: number = INSPECTION_PRIORITY.normal) {
|
||||
const ids = [...new Set(fileIds)]
|
||||
for (let i = 0; i < ids.length; i += 10_000) {
|
||||
await db.rawQuery(
|
||||
`INSERT INTO inspection_queue (file_id, priority)
|
||||
SELECT id, ?::smallint FROM unnest(?::bigint[]) AS id
|
||||
ON CONFLICT (file_id) DO UPDATE SET priority = least(inspection_queue.priority, excluded.priority)`,
|
||||
[priority, ids.slice(i, i + 10_000)]
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Tops BullMQ up to feedMax waiting jobs; returns how many were moved. */
|
||||
async feed(): Promise<number> {
|
||||
const queue = QueueService.collectionQueue
|
||||
const counts = await queue.getJobCounts('wait', 'prioritized')
|
||||
let room = this.feedMax - (counts.wait ?? 0) - (counts.prioritized ?? 0)
|
||||
let moved = 0
|
||||
while (room > 0) {
|
||||
const n = await db.transaction(async (trx) => {
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT file_id FROM inspection_queue ORDER BY priority, queued_at LIMIT ? FOR UPDATE SKIP LOCKED`,
|
||||
[Math.min(room, FEED_BATCH)]
|
||||
)
|
||||
if (rows.length === 0) return 0
|
||||
const ids = rows.map((row: { file_id: string }) => Number(row.file_id))
|
||||
// Added before the rows go: a failure leaves them queued (a job that went in anyway is deduplicated by its id).
|
||||
await queue.addBulk(ids.map((id: number) => ({ name: 'process-collection', data: { fileId: id }, opts: { jobId: `file-${id}` } })))
|
||||
await trx.rawQuery(`DELETE FROM inspection_queue WHERE file_id = ANY(?::bigint[])`, [ids])
|
||||
return ids.length
|
||||
})
|
||||
if (n === 0) break
|
||||
moved += n
|
||||
room -= n
|
||||
}
|
||||
return moved
|
||||
}
|
||||
|
||||
async size(): Promise<number> {
|
||||
const { rows } = await db.rawQuery(`SELECT count(*) AS n FROM inspection_queue`)
|
||||
return Number(rows[0].n)
|
||||
}
|
||||
}
|
||||
|
||||
export default new InspectionQueue()
|
||||
@@ -1,128 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
|
||||
/**
|
||||
* A release that is nothing more than one randomly named file (its own
|
||||
* filename stem). Such a file moves into the par2 set that names it exactly;
|
||||
* real groupings (par2, RAR, NZB, spot, subject, post id) are left alone.
|
||||
*/
|
||||
export const WEAK_RELEASE_SQL = (column: string) =>
|
||||
`EXISTS (SELECT 1 FROM releases wr WHERE wr.id = ${column} AND wr.source IN ('stem', 'weak-stem')
|
||||
AND wr.file_count <= 1 AND (wr.name !~ '[._ -]' OR NOT is_readable_name(wr.name)))`
|
||||
|
||||
/**
|
||||
* Whether `candidate` (e.g. from a par2 listing) should replace `current`:
|
||||
* the current name is random, or a single token without separators
|
||||
* (mOzyyxCyYUj, F8lv3w5agMDI3; too short for looksRandom), while the
|
||||
* candidate reads like a release name.
|
||||
*/
|
||||
export function isBetterName(current: string, candidate: string): boolean {
|
||||
if (looksRandom(candidate) || !/[._ -]/.test(candidate)) return false
|
||||
return looksRandom(current) || !/[._ -]/.test(current)
|
||||
}
|
||||
|
||||
/** Smaller files (nfo, sfv) collide too easily on size alone. */
|
||||
const MIN_SIZE_ONLY = 64 * 1024
|
||||
|
||||
/**
|
||||
* Gives binaries (and their files) the real filename that a par2 set lists
|
||||
* for them. See CONCEPT.md (D21).
|
||||
* - Exact: size and MD5 of the first 16 KiB both match a par2 entry. This
|
||||
* also puts the binary in that par2 set's release.
|
||||
* - Size only, when the fingerprint is unknown: within the binary's own
|
||||
* release, only when exactly one listed file has that size.
|
||||
*/
|
||||
class NameService {
|
||||
async resolve(binaryIds: number[]): Promise<number> {
|
||||
if (binaryIds.length === 0) return 0
|
||||
const exact = await db.rawQuery(
|
||||
`UPDATE binaries b SET real_name = m.name,
|
||||
release_id = CASE WHEN b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')} THEN m.release_id ELSE b.release_id END
|
||||
FROM (
|
||||
SELECT DISTINCT ON (b2.id) b2.id, rf.name, rf.release_id
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.size = b2.size AND rf.md5_16k = b2.md5_16k
|
||||
WHERE b2.id = ANY(?)
|
||||
ORDER BY b2.id, rf.release_id DESC
|
||||
) m
|
||||
WHERE b.id = m.id AND (b.real_name IS DISTINCT FROM m.name OR b.release_id IS NULL
|
||||
OR (b.release_id <> m.release_id AND ${WEAK_RELEASE_SQL('b.release_id')}))`,
|
||||
[binaryIds]
|
||||
)
|
||||
const bySize = await db.rawQuery(
|
||||
`UPDATE binaries b SET real_name = m.name
|
||||
FROM (
|
||||
SELECT b2.id, min(rf.name) AS name
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.release_id = b2.release_id AND rf.size = b2.size
|
||||
WHERE b2.id = ANY(?) AND b2.real_name IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
|
||||
GROUP BY b2.id
|
||||
HAVING count(DISTINCT rf.name) = 1
|
||||
) m
|
||||
WHERE b.id = m.id`,
|
||||
[binaryIds, MIN_SIZE_ONLY]
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE files f SET real_name = b.real_name,
|
||||
release_id = CASE WHEN f.release_id IS NULL OR ${WEAK_RELEASE_SQL('f.release_id')} THEN coalesce(b.release_id, f.release_id) ELSE f.release_id END
|
||||
FROM binaries b
|
||||
WHERE b.id = ANY(?) AND b.file_id = f.id
|
||||
AND (f.real_name IS DISTINCT FROM b.real_name OR (f.release_id IS NULL AND b.release_id IS NOT NULL)
|
||||
OR (f.release_id <> b.release_id AND ${WEAK_RELEASE_SQL('f.release_id')}))`,
|
||||
[binaryIds]
|
||||
)
|
||||
return (exact.rowCount ?? 0) + (bySize.rowCount ?? 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* When a par2 set arrives: the binaries it can name, i.e. those matching
|
||||
* one of its entries exactly (size + first-16-KiB MD5) and those in its
|
||||
* release. Selecting every binary of the listed sizes instead picked up
|
||||
* hundreds of thousands of equal-sized volumes per set (LOG.md 2026-09-28 (18)).
|
||||
*/
|
||||
async resolveForRelease(releaseId: number): Promise<number> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT b.id FROM release_files rf
|
||||
JOIN binaries b ON b.md5_16k = rf.md5_16k AND b.size = rf.size
|
||||
WHERE rf.release_id = ?
|
||||
UNION
|
||||
SELECT id FROM binaries WHERE release_id = ? AND size IS NOT NULL`,
|
||||
[releaseId, releaseId]
|
||||
)
|
||||
return this.resolve(rows.map((r: { id: string }) => Number(r.id)))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Repair for data from before files moved into the par2 set that names them
|
||||
* (used by `release:rename`): regroups exact matches out of random-named
|
||||
* single-file releases, then renames par2 releases that still carry a
|
||||
* random name after their listing. No downloads.
|
||||
*/
|
||||
export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; renamed: number }> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT b.id FROM binaries b
|
||||
JOIN release_files rf ON rf.md5_16k = b.md5_16k AND rf.size = b.size
|
||||
WHERE b.release_id IS DISTINCT FROM rf.release_id
|
||||
AND (b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')})`
|
||||
)
|
||||
const ids = rows.map((r: { id: string }) => Number(r.id))
|
||||
for (let i = 0; i < ids.length; i += 1000) await nameService.resolve(ids.slice(i, i + 1000))
|
||||
|
||||
const { par2ReleaseName } = await import('#lib/release/index')
|
||||
const listings = await db.rawQuery(
|
||||
`SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r
|
||||
JOIN release_files rf ON rf.release_id = r.id WHERE r.source <> 'spot' GROUP BY r.id, r.name`
|
||||
)
|
||||
let renamed = 0
|
||||
for (const release of listings.rows) {
|
||||
const better = par2ReleaseName(release.names)
|
||||
if (!better || better === release.name || !isBetterName(release.name, better)) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
renamed++
|
||||
}
|
||||
return { regrouped: ids.length, renamed }
|
||||
}
|
||||
|
||||
const nameService = new NameService()
|
||||
export default nameService
|
||||
@@ -1,141 +0,0 @@
|
||||
import nntpConfig from '#config/nntp'
|
||||
import { NNTP } from "#lib/nntp/nntp_js";
|
||||
import { parseGroupResponse, type GroupInfo } from '#lib/nntp/group'
|
||||
|
||||
class NntpService {
|
||||
private readonly poolSize: number;
|
||||
private allConnections: Set<any>;
|
||||
private idleConnections: any[];
|
||||
private waiters: ((conn: any) => void)[];
|
||||
/** Background work (archive walks): served only when no one else waits. */
|
||||
private lowWaiters: ((conn: any) => void)[] = [];
|
||||
private createdCount: number;
|
||||
|
||||
constructor(poolSize = 10) {
|
||||
this.poolSize = poolSize;
|
||||
this.allConnections = new Set();
|
||||
this.idleConnections = [];
|
||||
this.waiters = [];
|
||||
this.createdCount = 0;
|
||||
}
|
||||
|
||||
private async _createConnection() {
|
||||
const conn = new NNTP(nntpConfig.host, nntpConfig.port);
|
||||
try {
|
||||
await conn.connect();
|
||||
if (nntpConfig.user) {
|
||||
await conn.login(nntpConfig.user, nntpConfig.password?.release());
|
||||
}
|
||||
} catch (error) {
|
||||
// Free the slot, otherwise a failed connect shrinks the pool for good.
|
||||
this.createdCount--;
|
||||
this._destroy(conn);
|
||||
throw error;
|
||||
}
|
||||
this.allConnections.add(conn);
|
||||
return conn;
|
||||
}
|
||||
|
||||
/** A connection from the pool; with `low`, after every waiting normal caller. */
|
||||
public async acquire(low = false) {
|
||||
if (this.idleConnections.length > 0) {
|
||||
return this.idleConnections.pop();
|
||||
}
|
||||
|
||||
if (this.createdCount < this.poolSize) {
|
||||
this.createdCount++;
|
||||
console.log(`Creating new connection (${this.createdCount}/${this.poolSize}).`);
|
||||
return this._createConnection();
|
||||
}
|
||||
|
||||
return new Promise(resolve => (low ? this.lowWaiters : this.waiters).push(resolve));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a connection to the pool. Pass `failed` when a command on it
|
||||
* threw: the connection may be dead (e.g. closed by the server while idle),
|
||||
* so it is discarded instead of being handed to the next job.
|
||||
*/
|
||||
public release(conn: any, failed = false) {
|
||||
if (failed) {
|
||||
this.discard(conn);
|
||||
return;
|
||||
}
|
||||
const resolve = this.nextWaiter();
|
||||
if (resolve) {
|
||||
resolve(conn);
|
||||
} else {
|
||||
this.idleConnections.push(conn);
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes a broken connection and frees its slot for a new one. */
|
||||
public discard(conn: any) {
|
||||
this.allConnections.delete(conn);
|
||||
this.createdCount--;
|
||||
this._destroy(conn);
|
||||
|
||||
// Someone is waiting for a slot: open a fresh connection for them.
|
||||
const resolve = this.nextWaiter();
|
||||
if (resolve) {
|
||||
this.createdCount++;
|
||||
this._createConnection().then(resolve, () => this.waiters.unshift(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
private nextWaiter() {
|
||||
return this.waiters.shift() ?? this.lowWaiters.shift();
|
||||
}
|
||||
|
||||
private _destroy(conn: any) {
|
||||
try {
|
||||
conn.sock?.destroy();
|
||||
} catch {
|
||||
// already closed
|
||||
}
|
||||
}
|
||||
|
||||
/** Selects a group and returns its correctly parsed article range. */
|
||||
public async selectGroup(conn: any, name: string): Promise<GroupInfo> {
|
||||
const { resp } = await conn.group(name)
|
||||
return parseGroupResponse(resp)
|
||||
}
|
||||
|
||||
/**
|
||||
* Rejects when `promise` hasn't settled within `ms`. nntp-js has no
|
||||
* timeouts: a connection the server stops answering (still ESTABLISHED,
|
||||
* nothing in flight) leaves its command waiting forever. The caller must
|
||||
* then release the connection as failed, so it's destroyed.
|
||||
*/
|
||||
public timed<T>(promise: Promise<T>, what: string, ms = nntpConfig.commandTimeoutMs): Promise<T> {
|
||||
let timer: NodeJS.Timeout
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`NNTP ${what}: no answer within ${ms / 1000} s`)), ms)
|
||||
})
|
||||
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer))
|
||||
}
|
||||
|
||||
/** Fetches a raw (still dot-stuffed) article body as a Buffer. */
|
||||
public async fetchBody(conn: any, messageId: string): Promise<Buffer> {
|
||||
const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
|
||||
const { data } = await conn.body(id)
|
||||
return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
|
||||
}
|
||||
|
||||
public async shutdown() {
|
||||
console.log('Shutting down all connections in the pool.');
|
||||
// A connection the server already closed may never answer QUIT.
|
||||
const quit = (conn: any) =>
|
||||
Promise.race([conn.quit(), new Promise((resolve) => setTimeout(resolve, 2000).unref())])
|
||||
.catch(() => {})
|
||||
.finally(() => this._destroy(conn));
|
||||
await Promise.all([...this.allConnections].map(quit));
|
||||
this.allConnections.clear();
|
||||
this.idleConnections.length = 0;
|
||||
this.waiters.length = 0;
|
||||
this.lowWaiters.length = 0;
|
||||
this.createdCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
export default new NntpService(nntpConfig.poolSize);
|
||||
@@ -1,229 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { type ReleaseIdentity } from '#lib/release/index'
|
||||
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
||||
import { parseSubject } from '#lib/subject/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
import FileStore from '#services/FileStore'
|
||||
import { isArchiveStart } from '#lib/content/index'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
|
||||
export interface NzbImportResult {
|
||||
releaseId: number
|
||||
releaseName: string
|
||||
listed: number
|
||||
known: number
|
||||
added: number
|
||||
password: boolean
|
||||
}
|
||||
|
||||
/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
|
||||
export function nzbFileName(subject: string): string {
|
||||
const parsed = parseSubject(subject)
|
||||
const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1]
|
||||
if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
|
||||
return unquotedFileName(subject)
|
||||
}
|
||||
|
||||
/**
|
||||
* A filename from a subject without quotes, as some spot NZBs have:
|
||||
* `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`,
|
||||
* `06 - Song.flac Artist - Album (2020) (01/35)`. Counters, "yEnc" and a
|
||||
* trailing size are dropped; then the name ends at the first extension
|
||||
* followed by a space.
|
||||
*/
|
||||
export function unquotedFileName(subject: string): string {
|
||||
const cleaned = subject
|
||||
.replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ')
|
||||
.replace(/\s\d+\s*$/, ' ')
|
||||
.replace(/(\s+-)+\s*$/, '')
|
||||
.replace(/^\s*-\s+/, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned)
|
||||
const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned
|
||||
return name.replace(/\s+-$/, '').trim() || subject.trim()
|
||||
}
|
||||
|
||||
export interface NzbImportOptions {
|
||||
/**
|
||||
* For NZBs inside an archive: the entry's name. It keeps several NZBs of
|
||||
* one archive apart and names their release when the NZB has no name.
|
||||
*/
|
||||
entryName?: string
|
||||
/** Move the posted file into the release the NZB describes (default true). */
|
||||
moveContainer?: boolean
|
||||
/** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */
|
||||
name?: string
|
||||
/** Release source for new releases (default `nzb`). */
|
||||
source?: ReleaseIdentity['source']
|
||||
/**
|
||||
* The NZB is authoritative for which files form the release and what it is
|
||||
* called (Spotnet spots): its files get the NZB's own release, named
|
||||
* `name`, also when name rules had put them in another release. Files
|
||||
* another trusted NZB claimed first stay where they are.
|
||||
*/
|
||||
trusted?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26).
|
||||
* Files it lists that are already known (by first segment) keep their
|
||||
* release, which the NZB then joins; the others are added as files, so posts
|
||||
* spread over groups we do not index still become complete.
|
||||
*/
|
||||
class NzbImportService {
|
||||
async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise<NzbImportResult | null> {
|
||||
const nzb = parseNzb(xml)
|
||||
if (!nzb || nzb.files.length === 0) return null
|
||||
|
||||
const entries = nzb.files.map((f) => ({
|
||||
file: f,
|
||||
name: nzbFileName(f.subject),
|
||||
firstId: f.segments[0].id,
|
||||
}))
|
||||
|
||||
const added: { id: number; filename: string }[] = []
|
||||
const result = await db.transaction(async (trx) => {
|
||||
const known = await trx
|
||||
.from('files')
|
||||
.whereIn('first_message_id', entries.map((e) => e.firstId))
|
||||
.select('id', 'release_id', 'first_message_id')
|
||||
const knownIds = new Set(known.map((f) => f.first_message_id))
|
||||
|
||||
// Join the release the listed files are already in, else the posted NZB's own, else a new one.
|
||||
const counts = new Map<number, number>()
|
||||
for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
|
||||
let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
|
||||
const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
|
||||
const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}`
|
||||
const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
|
||||
|
||||
// Releases of other trusted NZBs: their files are not taken over.
|
||||
let claimed = new Set<number>()
|
||||
if (options.trusted) {
|
||||
// Only while that NZB still stands: a disposed spot's files go to its correction.
|
||||
const holders = await trx
|
||||
.from('releases as r')
|
||||
.whereIn('r.id', [...counts.keys()])
|
||||
.where('r.source', 'spot')
|
||||
.whereNot('r.release_key', key)
|
||||
.whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy']))
|
||||
.select('r.id')
|
||||
claimed = new Set(holders.map((r) => Number(r.id)))
|
||||
// Join another spot's release when it holds most of these files (a repost); else use our own.
|
||||
if (releaseId && !claimed.has(releaseId)) releaseId = undefined
|
||||
if (releaseId) claimed.delete(releaseId)
|
||||
}
|
||||
if (!releaseId) {
|
||||
const ids = await IngestService.upsertReleases(trx.knexClient, [
|
||||
{
|
||||
release: { key, name: title, source: options.source ?? 'nzb' },
|
||||
poster: nzbFile.poster,
|
||||
groups: nzbFile.groups ?? [],
|
||||
postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined,
|
||||
},
|
||||
])
|
||||
releaseId = ids.get(key)!
|
||||
}
|
||||
|
||||
const now = new Date()
|
||||
const missing = entries.filter((e) => !knownIds.has(e.firstId))
|
||||
for (let i = 0; i < missing.length; i += 500) {
|
||||
added.push(
|
||||
...(await FileStore.insert(
|
||||
missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
|
||||
file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
|
||||
filename: name,
|
||||
poster: file.poster || nzbFile.poster,
|
||||
date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
|
||||
parts: file.segments.length,
|
||||
size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
|
||||
segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })),
|
||||
groups: JSON.stringify(file.groups),
|
||||
release_id: releaseId,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
})),
|
||||
trx
|
||||
))
|
||||
)
|
||||
}
|
||||
// The poster's NZB says which files form the post: known files join this
|
||||
// release, also when name rules had split them over several.
|
||||
const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id)))
|
||||
const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))]
|
||||
await trx
|
||||
.from('files')
|
||||
.whereIn('id', moving.map((f) => f.id))
|
||||
.update({ release_id: releaseId })
|
||||
if (previous.length) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`,
|
||||
[releaseId, releaseId, releaseId]
|
||||
)
|
||||
await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete()
|
||||
}
|
||||
// The NZB file joins the release it describes, unless it is part of a real release already.
|
||||
const own = nzbFile.release_id ? Number(nzbFile.release_id) : null
|
||||
if (own !== releaseId && options.moveContainer !== false) {
|
||||
const ownRelease = own ? await trx.from('releases').where('id', own).first() : null
|
||||
if (!ownRelease || ownRelease.file_count <= 1) {
|
||||
await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId })
|
||||
if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete()
|
||||
}
|
||||
}
|
||||
|
||||
const release = await trx.from('releases').where('id', releaseId).first()
|
||||
const update: Record<string, unknown> = {}
|
||||
if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' })
|
||||
if (!release.file_total) update.file_total = entries.length
|
||||
if (options.trusted) {
|
||||
// Only its own release: a repost joining another spot's release keeps that one's title.
|
||||
if (release.release_key === key && release.name !== title) update.name = title
|
||||
if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source
|
||||
} else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) {
|
||||
update.name = title
|
||||
}
|
||||
if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
|
||||
|
||||
const result = {
|
||||
releaseId,
|
||||
releaseName: (update.name as string) ?? release.name,
|
||||
listed: entries.length,
|
||||
known: known.length,
|
||||
added: missing.length,
|
||||
password: Boolean(nzb.meta.password),
|
||||
}
|
||||
logger.info(
|
||||
`NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` +
|
||||
`${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.`
|
||||
)
|
||||
return result
|
||||
})
|
||||
// Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
|
||||
const archives = added.filter((f) => isArchiveStart(f.filename))
|
||||
if (archives.length) {
|
||||
await InspectionQueue.add(archives.map((f) => Number(f.id)))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
|
||||
private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string {
|
||||
if (nzb.meta.name) return nzb.meta.name
|
||||
const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : ''
|
||||
if (entry && !looksRandom(entry)) return entry
|
||||
const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? ''))
|
||||
if (own && !looksRandom(own)) return own
|
||||
const largest = nzb.files
|
||||
.map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) }))
|
||||
.sort((a, b) => b.bytes - a.bytes)[0]
|
||||
return fileStem(largest?.name ?? own)
|
||||
}
|
||||
}
|
||||
|
||||
export default new NzbImportService()
|
||||
@@ -1,133 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
|
||||
import type { Par2Set } from '#lib/par2/index'
|
||||
import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
|
||||
import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'
|
||||
|
||||
/** Size-only matching below this is too ambiguous. */
|
||||
const SIZE_ONLY_MIN = 1024 * 1024
|
||||
|
||||
function countBy<T>(values: T[]): Map<T, number> {
|
||||
const counts = new Map<T, number>()
|
||||
for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1)
|
||||
return counts
|
||||
}
|
||||
|
||||
export interface Par2Result {
|
||||
releaseId: number
|
||||
releaseName: string
|
||||
listed: number
|
||||
matched: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognises releases of obfuscated posts from their par2 files: the par2
|
||||
* lists the real name and exact size of every file in the set, and
|
||||
* body-derived binaries carry their exact size. See CONCEPT.md (D16).
|
||||
*/
|
||||
class Par2Service {
|
||||
async attach(binary: any, set: Par2Set): Promise<Par2Result> {
|
||||
const result = await db.transaction(async (trx) => {
|
||||
// Exact: size and first-16-KiB MD5 of a listed file. Only these may pull
|
||||
// the set into a release the files already belong to; size alone is
|
||||
// ambiguous across posts (standard volume sizes recur).
|
||||
const exact = await trx
|
||||
.from('binaries')
|
||||
.whereIn(['size', 'md5_16k'] as any, set.files.map((f) => [f.length, f.md5_16k]) as any)
|
||||
.select('id', 'release_id')
|
||||
|
||||
// Size only, for files without a fingerprint: the size must be listed
|
||||
// once in this set, be large, and belong to exactly one binary nearby.
|
||||
const listedOnce = [...countBy(set.files.map((f) => f.length))]
|
||||
.filter(([size, n]) => n === 1 && size >= SIZE_ONLY_MIN)
|
||||
.map(([size]) => size)
|
||||
const bySize = listedOnce.length
|
||||
? await trx
|
||||
.from('binaries')
|
||||
.whereIn('size', listedOnce)
|
||||
.whereNull('md5_16k')
|
||||
.whereRaw('abs(extract(epoch FROM (posted_at - ?::timestamptz))) < ?', [
|
||||
binary.posted_at,
|
||||
PAR2_MATCH_WINDOW_SECONDS,
|
||||
])
|
||||
.select('id', 'release_id', 'size')
|
||||
: []
|
||||
const sizeCounts = countBy(bySize.map((b) => Number(b.size)))
|
||||
const fuzzy = bySize.filter((b) => sizeCounts.get(Number(b.size)) === 1)
|
||||
const candidates = [...exact, ...fuzzy]
|
||||
|
||||
// Prefer a release the exactly matched files are already in (e.g.
|
||||
// obfuscated volumes grouped by their stem); otherwise the set's own.
|
||||
const counts = countBy(exact.filter((c) => c.release_id).map((c) => Number(c.release_id)))
|
||||
let releaseId = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
|
||||
const identity = par2ReleaseIdentity(
|
||||
set.recoverySetId,
|
||||
set.files.map((f) => f.name)
|
||||
)
|
||||
if (!releaseId) {
|
||||
const ids = await IngestService.upsertReleases(trx.knexClient, [
|
||||
{
|
||||
release: identity,
|
||||
poster: binary.poster,
|
||||
weakIdentity: true,
|
||||
groups: binary.groups,
|
||||
postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
|
||||
},
|
||||
])
|
||||
releaseId = ids.get(identity.key)!
|
||||
}
|
||||
|
||||
await trx
|
||||
.knexClient('release_files')
|
||||
.insert(
|
||||
set.files.map((f) => ({
|
||||
release_id: releaseId,
|
||||
par2_file_id: f.fileId,
|
||||
name: f.name,
|
||||
size: f.length,
|
||||
md5_16k: f.md5_16k,
|
||||
md5: f.md5,
|
||||
set_id: set.recoverySetId,
|
||||
}))
|
||||
)
|
||||
.onConflict(['release_id', 'par2_file_id'])
|
||||
.ignore()
|
||||
|
||||
const matchIds = candidates.filter((c) => !c.release_id).map((c) => Number(c.id))
|
||||
await trx.from('binaries').whereIn('id', [binary.id, ...matchIds]).whereNull('release_id').update({ release_id: releaseId })
|
||||
// Exact matches sitting alone in a random-named release (one file of a
|
||||
// season, each posted under its own random name) join the set's release.
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET release_id = ? WHERE b.id = ANY(?) AND b.release_id <> ? AND ${WEAK_RELEASE_SQL('b.release_id')}`,
|
||||
[releaseId, exact.map((c) => Number(c.id)), releaseId]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE files f SET release_id = b.release_id FROM binaries b
|
||||
WHERE b.file_id = f.id AND b.release_id = ? AND f.release_id IS DISTINCT FROM b.release_id`,
|
||||
[releaseId]
|
||||
)
|
||||
|
||||
// The set may have joined a release named after one random filename.
|
||||
const release = await trx.from('releases').where('id', releaseId).first()
|
||||
let releaseName = release?.name
|
||||
const better = par2ReleaseName(set.files.map((f) => f.name))
|
||||
// A spot's title is kept (CONCEPT.md D29).
|
||||
if (releaseName && better && release.source !== 'spot' && isBetterName(releaseName, better)) {
|
||||
await trx.from('releases').where('id', releaseId).update({ name: better })
|
||||
releaseName = better
|
||||
}
|
||||
logger.info(
|
||||
`Par2 set ${set.recoverySetId.slice(0, 8)} → release ${releaseId} "${releaseName}": ` +
|
||||
`${set.files.length} files listed, ${matchIds.length} binaries matched by size.`
|
||||
)
|
||||
return { releaseId, releaseName, listed: set.files.length, matched: matchIds.length }
|
||||
})
|
||||
|
||||
// Real names for every known file of the set (exact fingerprint, or size within the release).
|
||||
await NameService.resolveForRelease(result.releaseId)
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
export default new Par2Service()
|
||||
@@ -1,65 +0,0 @@
|
||||
import { Queue, type DefaultJobOptions } from 'bullmq'
|
||||
import queueConfig from '#config/queue'
|
||||
import gcConfig from '#config/gc'
|
||||
|
||||
const defaultJobOptions: DefaultJobOptions = {
|
||||
// Transient NNTP errors (e.g. a pooled connection the server closed) are retried.
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
removeOnComplete: {
|
||||
age: gcConfig.jobs.completedAgeSeconds,
|
||||
count: gcConfig.jobs.completedMaxCount,
|
||||
},
|
||||
removeOnFail: { age: gcConfig.jobs.failedAgeSeconds },
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues are created (and connect to Redis) on first use. Ace imports every
|
||||
* command at boot, so eager queues would keep one-off commands from exiting.
|
||||
*/
|
||||
class QueueService {
|
||||
private readonly queues = new Map<string, Queue>()
|
||||
|
||||
private queue(name: string): Queue {
|
||||
let queue = this.queues.get(name)
|
||||
if (!queue) {
|
||||
queue = new Queue(name, { connection: queueConfig.connection, defaultJobOptions })
|
||||
this.queues.set(name, queue)
|
||||
}
|
||||
return queue
|
||||
}
|
||||
|
||||
get nntpFetchQueue() {
|
||||
return this.queue('nntp-fetch-queue')
|
||||
}
|
||||
get fileQueue() {
|
||||
return this.queue('file-queue')
|
||||
}
|
||||
get bodyQueue() {
|
||||
return this.queue('body-queue')
|
||||
}
|
||||
get collectionQueue() {
|
||||
return this.queue('collection-queue')
|
||||
}
|
||||
/** Complete archive listings, at low priority (ArchiveWalkService). */
|
||||
get archiveWalkQueue() {
|
||||
return this.queue('archive-walk-queue')
|
||||
}
|
||||
|
||||
/** Queues no longer used by the pipeline, kept here so GC can empty them. */
|
||||
get legacy(): Queue[] {
|
||||
return [this.queue('header-queue')]
|
||||
}
|
||||
|
||||
get all(): Queue[] {
|
||||
return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue, this.archiveWalkQueue]
|
||||
}
|
||||
|
||||
/** Closes the queues that were opened. */
|
||||
async closeAll() {
|
||||
await Promise.all([...this.queues.values()].map((queue) => queue.close()))
|
||||
this.queues.clear()
|
||||
}
|
||||
}
|
||||
|
||||
export default new QueueService()
|
||||
@@ -1,25 +0,0 @@
|
||||
import { createRequire } from 'module'
|
||||
import redisConfig from '#config/redis'
|
||||
|
||||
const require = createRequire(import.meta.url)
|
||||
const IORedis = require('ioredis')
|
||||
|
||||
/**
|
||||
* Connects on first use. Ace imports every command at boot, so an eager
|
||||
* connection would keep even `node ace list` from exiting.
|
||||
*/
|
||||
class RedisService {
|
||||
private instance?: any
|
||||
|
||||
public get client(): any {
|
||||
this.instance ??= new IORedis(redisConfig.connections.main)
|
||||
return this.instance
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.instance) await this.instance.quit()
|
||||
this.instance = undefined
|
||||
}
|
||||
}
|
||||
|
||||
export default new RedisService()
|
||||
@@ -1,35 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Noted releases taken per transaction. */
|
||||
const BATCH = 2000
|
||||
|
||||
/**
|
||||
* Release completeness (D24) for releases whose files or par2 listing
|
||||
* changed: triggers only note them in release_dirty (a release can be noted
|
||||
* thousands of times between sweeps), and gc:scheduler recomputes each
|
||||
* noted release once every RELEASE_COMPLETE_INTERVAL_MS (LOG.md 2026-10-06 (6)).
|
||||
*/
|
||||
class ReleaseCompleteness {
|
||||
/** Recomputes all noted releases; returns how many distinct releases were checked. */
|
||||
async sweep(): Promise<number> {
|
||||
let checked = 0
|
||||
for (;;) {
|
||||
const done = await db.transaction(async (trx) => {
|
||||
// Taken and recomputed in one transaction: a failure leaves the notes for the next sweep.
|
||||
const { rows } = await trx.rawQuery(
|
||||
`DELETE FROM release_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM release_dirty LIMIT ?)) RETURNING release_id`,
|
||||
[BATCH]
|
||||
)
|
||||
const ids: number[] = [...new Set<number>(rows.map((row: { release_id: string }) => Number(row.release_id)))]
|
||||
if (ids.length > 0) {
|
||||
await trx.rawQuery(`SELECT recompute_release_complete(id) FROM unnest(?::bigint[]) AS id`, [ids])
|
||||
}
|
||||
checked += ids.length
|
||||
return rows.length < BATCH
|
||||
})
|
||||
if (done) return checked
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new ReleaseCompleteness()
|
||||
@@ -1,42 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/**
|
||||
* Uploads split per poster (CONCEPT.md D35). Some uploaders post every
|
||||
* volume of a RAR set under a different poster; the post-id grouping then
|
||||
* makes one release per volume, all with the same name
|
||||
* (`Masters.of.the.Universe.1987…`: 191 one-file releases of one 199-volume
|
||||
* set). Such releases are one upload when all their files share one stem and
|
||||
* no file name repeats (a repost would repeat them, another upload would
|
||||
* bring another stem); their files move into the oldest of them, and GC
|
||||
* deletes the emptied ones.
|
||||
*/
|
||||
class ReleaseMergeService {
|
||||
async mergeSplitPosts({ dryRun = false } = {}): Promise<{ names: number; releases: number }> {
|
||||
const { rows } = await db.rawQuery(`
|
||||
WITH dup AS (
|
||||
SELECT name FROM releases
|
||||
WHERE file_count > 0 AND source = 'post' AND is_readable_name(name)
|
||||
GROUP BY name HAVING count(*) > 1
|
||||
)
|
||||
SELECT r.name, array_agg(DISTINCT r.id ORDER BY r.id) AS ids
|
||||
FROM dup JOIN releases r ON r.name = dup.name AND r.file_count > 0 AND r.source = 'post'
|
||||
JOIN files f ON f.release_id = r.id
|
||||
GROUP BY r.name
|
||||
HAVING count(DISTINCT name_stem(coalesce(f.real_name, f.filename))) = 1
|
||||
AND count(f.id) = count(DISTINCT coalesce(f.real_name, f.filename))
|
||||
AND max(r.posted_at) - min(r.posted_at) < interval '30 days'`)
|
||||
let releases = 0
|
||||
for (const { name, ids } of rows as { name: string; ids: (string | number)[] }[]) {
|
||||
const [target, ...others] = ids.map(Number)
|
||||
releases += others.length
|
||||
if (dryRun) continue
|
||||
await db.from('files').whereIn('release_id', others).update({ release_id: target })
|
||||
await db.from('binaries').whereIn('release_id', others).update({ release_id: target })
|
||||
logger.info(`Merged ${others.length + 1} releases of one upload split per poster into ${target} "${name}".`)
|
||||
}
|
||||
return { names: rows.length, releases }
|
||||
}
|
||||
}
|
||||
|
||||
export default new ReleaseMergeService()
|
||||
@@ -1,168 +0,0 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { Job, Worker } from 'bullmq'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import queueConfig from '#config/queue'
|
||||
import nntpConfig from '#config/nntp'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NntpService from '#services/NntpService'
|
||||
import ContentService, { describeContent } from '#services/ContentService'
|
||||
import ArchiveWalkService from '#services/ArchiveWalkService'
|
||||
import { isMissingArticle, type ContentTask, type InspectProbe, type InspectTask, type WalkProbe } from '#lib/content/probe'
|
||||
|
||||
/** Inspection first; archive walks (D40) only when it has nothing waiting. */
|
||||
const QUEUES = ['collection-queue', 'archive-walk-queue'] as const
|
||||
type QueueName = (typeof QUEUES)[number]
|
||||
|
||||
/** A node has this long to post a job's result; after that the job goes back to the queue (BullMQ's stalled check). */
|
||||
const LOCK_MS = 10 * 60 * 1000
|
||||
|
||||
export interface RemoteJob {
|
||||
queue: QueueName
|
||||
jobId: string
|
||||
/** The job's lock: whoever holds it may complete the job. */
|
||||
token: string
|
||||
task: ContentTask
|
||||
}
|
||||
|
||||
export class JobLost extends Error {}
|
||||
export class ArticleRelayOff extends Error {}
|
||||
|
||||
/**
|
||||
* Content inspection and archive walks on seeding nodes (CONCEPT.md D41).
|
||||
* Nodes take jobs from the same queues worker:collection works on: a lease
|
||||
* locks a job (BullMQ's manual processing) and gives the node a task that
|
||||
* was prepared from the database; the node probes it, fetching articles
|
||||
* through the relay (articles()), and posts the result, which is applied
|
||||
* here. A job whose node goes quiet is unlocked after LOCK_MS and picked up
|
||||
* again by whoever comes next.
|
||||
*/
|
||||
class RemoteContentService {
|
||||
private readonly workers = new Map<QueueName, Worker>()
|
||||
|
||||
/** Up to `max` jobs for a node, each locked for LOCK_MS. */
|
||||
async lease(nodeId: number, max: number): Promise<RemoteJob[]> {
|
||||
const jobs: RemoteJob[] = []
|
||||
for (const queue of QUEUES) {
|
||||
while (jobs.length < max) {
|
||||
const token = `node-${nodeId}:${randomUUID()}`
|
||||
const job = await this.worker(queue).getNextJob(token, { block: false })
|
||||
if (!job) break
|
||||
try {
|
||||
const task = await this.prepare(queue, job)
|
||||
if (!task) {
|
||||
await job.moveToCompleted(null, token, false)
|
||||
continue
|
||||
}
|
||||
// What applying needs, kept with the job: the node's copy of the task isn't trusted.
|
||||
if (task.kind === 'inspect') await job.updateData({ ...job.data, remote: { fileId: task.fileId, name: task.name } })
|
||||
jobs.push({ queue, jobId: job.id!, token, task })
|
||||
} catch (error: any) {
|
||||
await job.moveToFailed(error, token, false).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
return jobs
|
||||
}
|
||||
|
||||
/** Applies a node's result and completes the job. */
|
||||
async finish(queue: QueueName, jobId: string, token: string, result: InspectProbe | WalkProbe) {
|
||||
const job = await this.job(queue, jobId)
|
||||
let summary: unknown
|
||||
if (queue === 'collection-queue') {
|
||||
const remote = job.data.remote
|
||||
if (!remote) throw new JobLost(`Job ${jobId} was not leased to a node`)
|
||||
const task: InspectTask = { kind: 'inspect', fileId: Number(remote.fileId), name: String(remote.name), segments: [], sevenZip: null }
|
||||
const content = await ContentService.apply(task, result as InspectProbe)
|
||||
for (const line of describeContent(task.name, content)) logger.info(`[node] ${line}`)
|
||||
summary = content.contentType
|
||||
} else {
|
||||
const outcome = await ArchiveWalkService.apply(Number(job.data.releaseId), result as WalkProbe)
|
||||
logger.info(
|
||||
`[node] Archive walk of release ${outcome.releaseId}: ${outcome.before} → ${outcome.entries} entries from ${outcome.segments} segments` +
|
||||
(outcome.complete ? '' : ` (stopped: ${outcome.stopped})`) +
|
||||
(outcome.renamed ? `, renamed to "${outcome.renamed}"` : '')
|
||||
)
|
||||
summary = outcome
|
||||
}
|
||||
await this.locked(() => job.moveToCompleted(summary, token, false))
|
||||
}
|
||||
|
||||
/** The probe failed on the node (e.g. a missing article): retried like a local failure. */
|
||||
async fail(queue: QueueName, jobId: string, token: string, error: string) {
|
||||
const job = await this.job(queue, jobId)
|
||||
// A missing first article from a node that predates the probe's own handling: a result,
|
||||
// not something to retry (each try costs Eweka ~450 ms; LOG.md 2026-10-06 (14)).
|
||||
if (queue === 'collection-queue' && job.data.remote && isMissingArticle(error)) {
|
||||
return this.finish(queue, jobId, token, { size: 0, contentType: 'missing', md5_16k: null, yencName: '' })
|
||||
}
|
||||
await this.locked(() => job.moveToFailed(new Error(`node: ${error}`), token, false))
|
||||
}
|
||||
|
||||
/** A stopping node gives its jobs back, without using up an attempt. */
|
||||
async release(queue: QueueName, jobId: string, token: string) {
|
||||
const job = await this.job(queue, jobId)
|
||||
await this.locked(() => job.moveToWait(token))
|
||||
}
|
||||
|
||||
/** One article body for a relay node, on this process's NNTP pool. Null when the server doesn't have it. */
|
||||
async article(messageId: string): Promise<Buffer | null> {
|
||||
// NNTP_POOL_SIZE 0 (SEED_ARTICLE_CONNECTIONS=0): no connections for this; acquire() would wait forever.
|
||||
if (nntpConfig.poolSize <= 0) throw new ArticleRelayOff('Article relay is off (SEED_ARTICLE_CONNECTIONS=0)')
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
|
||||
} catch (error: any) {
|
||||
// 430 no such article: a fact about the article, the connection is fine.
|
||||
if (/^4\d\d/.test(String(error.message))) return null
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes the lease workers' Redis connections (tests; a server just exits). */
|
||||
async close() {
|
||||
await Promise.all([...this.workers.values()].map((worker) => worker.close()))
|
||||
this.workers.clear()
|
||||
}
|
||||
|
||||
isQueue(name: unknown): name is QueueName {
|
||||
return QUEUES.includes(name as QueueName)
|
||||
}
|
||||
|
||||
private async prepare(queue: QueueName, job: Job): Promise<ContentTask | null> {
|
||||
if (queue === 'collection-queue') return ContentService.prepare(Number(job.data.fileId))
|
||||
const prepared = await ArchiveWalkService.prepare(Number(job.data.releaseId))
|
||||
return 'task' in prepared ? prepared.task : null
|
||||
}
|
||||
|
||||
private async job(queue: QueueName, jobId: string): Promise<Job> {
|
||||
const job = await Job.fromId(queue === 'collection-queue' ? QueueService.collectionQueue : QueueService.archiveWalkQueue, jobId)
|
||||
if (!job) throw new JobLost(`Job ${jobId} is gone`)
|
||||
return job
|
||||
}
|
||||
|
||||
/** BullMQ refuses a job whose lock another holder has now: the node's result came too late. */
|
||||
private async locked(action: () => Promise<unknown>) {
|
||||
try {
|
||||
await action()
|
||||
} catch (error: any) {
|
||||
if (/lock|not in the active state|Missing key/i.test(String(error.message))) throw new JobLost(error.message)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** A worker that only hands out jobs (no processor, never runs). */
|
||||
private worker(queue: QueueName): Worker {
|
||||
let worker = this.workers.get(queue)
|
||||
if (!worker) {
|
||||
worker = new Worker(queue, null, { connection: queueConfig.connection, lockDuration: LOCK_MS, autorun: false })
|
||||
this.workers.set(queue, worker)
|
||||
}
|
||||
return worker
|
||||
}
|
||||
}
|
||||
|
||||
export default new RemoteContentService()
|
||||
@@ -1,421 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { buildNzb } from '#lib/nzb/index'
|
||||
import SpotService from '#services/SpotService'
|
||||
import { segmentsOf } from '#services/FileStore'
|
||||
import { spotCategoryLabels } from '#lib/spotnet/index'
|
||||
|
||||
/**
|
||||
* `known`: we have the password (it's in our NZB, so SABnzbd/NZBGet unpack
|
||||
* without asking); `missing`: the archive is encrypted and we don't have
|
||||
* it; `none`: no password needed as far as we know.
|
||||
*/
|
||||
export type PasswordState = 'known' | 'missing' | 'none'
|
||||
|
||||
export function passwordState(encrypted: boolean | null, hasPassword: boolean): PasswordState {
|
||||
if (hasPassword) return 'known'
|
||||
return encrypted ? 'missing' : 'none'
|
||||
}
|
||||
|
||||
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
||||
|
||||
export interface SearchParams {
|
||||
q?: string
|
||||
/** Only releases whose name is readable (not an obfuscated token). */
|
||||
readable?: boolean
|
||||
/** Only releases that have all their files. */
|
||||
complete?: boolean
|
||||
/** Hide releases that need a password we don't have. */
|
||||
hideEncrypted?: boolean
|
||||
source?: string
|
||||
poster?: string
|
||||
sort?: SortOrder
|
||||
page?: number
|
||||
perPage?: number
|
||||
/** Newznab: exact category ids (already expanded from parents). */
|
||||
categories?: number[]
|
||||
/** Newznab: leave out releases known to be missing files. */
|
||||
notIncomplete?: boolean
|
||||
maxAgeDays?: number
|
||||
/** TV: season and episode numbers, matched as S01E02 or 1x02 in the name. */
|
||||
season?: number
|
||||
episode?: number
|
||||
/** Overrides `page` (Newznab uses offsets). */
|
||||
offset?: number
|
||||
/** Leave out releases that are only an encrypted container (nzbe). */
|
||||
hideContainers?: boolean
|
||||
/** Include incomplete releases that consist of par2 files only (hidden by default). */
|
||||
includePar2Only?: boolean
|
||||
/** Size bounds in bytes (releases without a known size never match). */
|
||||
minSize?: number
|
||||
maxSize?: number
|
||||
/** File extensions (without the dot); a release matches if a file or archive entry has one of them (`releases.extensions`). */
|
||||
extensions?: string[]
|
||||
/** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
|
||||
countLimit?: number
|
||||
}
|
||||
|
||||
const SORTS: Record<SortOrder, string> = {
|
||||
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
|
||||
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
|
||||
largest: 'r.size DESC, r.id DESC',
|
||||
files: 'r.file_count DESC, r.id DESC',
|
||||
name: 'r.name ASC, r.id ASC',
|
||||
}
|
||||
|
||||
/**
|
||||
* Shown in search: releases with files, and Spotnet listings whose NZB is
|
||||
* fetched on first download (CONCEPT.md D29). Matches the partial indexes.
|
||||
*/
|
||||
const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')`
|
||||
|
||||
/** Incomplete, and every file is a par2 file: nothing to download but repair data. */
|
||||
const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS (
|
||||
SELECT 1 FROM files f LEFT JOIN file_details d ON d.file_id = f.id WHERE f.release_id = r.id
|
||||
AND d.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
|
||||
|
||||
/** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */
|
||||
function pattern(word: string): string {
|
||||
const escaped = word.replace(/[\\%]/g, (c) => `\\${c}`)
|
||||
return `%${escaped.replace(/[._\- ]+/g, '_')}%`
|
||||
}
|
||||
|
||||
/**
|
||||
* As SQL's search_normalize(): lowercase, and dots, dashes, underscores and
|
||||
* spaces as one space. `The.Block.AU` and `the block au` search the same.
|
||||
*/
|
||||
export function searchNormalize(text: string): string {
|
||||
return text.toLowerCase().replace(/[._\s-]+/g, ' ').trim()
|
||||
}
|
||||
|
||||
/** How long the web UI's totals are reused. */
|
||||
const STATS_TTL_MS = 60 * 1000
|
||||
|
||||
/**
|
||||
* Counting stops here: a broad search needn't count every match. Each
|
||||
* counted row is a random heap read, so the web UI counts less (it only
|
||||
* shows "1,000+"); Newznab clients page by this total and keep the default.
|
||||
*/
|
||||
const COUNT_LIMIT = 10_000
|
||||
|
||||
/**
|
||||
* Searching and browsing releases for the web interface (CONCEPT.md D22).
|
||||
*/
|
||||
class SearchService {
|
||||
async search(params: SearchParams) {
|
||||
const where: string[] = [
|
||||
VISIBLE_SQL,
|
||||
// Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
|
||||
`NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status IN ('removed', 'dummy'))`,
|
||||
]
|
||||
const bindings: unknown[] = []
|
||||
|
||||
// Every word must occur in the release's search text: its name, archive
|
||||
// listing and file names (CONCEPT.md D33), one trigram index for all.
|
||||
const words = (params.q ?? '').trim().split(/\s+/).map(searchNormalize).filter(Boolean).slice(0, 8)
|
||||
for (const word of words) {
|
||||
where.push(`r.search_text LIKE ?`)
|
||||
bindings.push(`%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`)
|
||||
}
|
||||
if (params.readable) where.push('is_readable_name(r.name)')
|
||||
if (params.minSize !== undefined) {
|
||||
where.push('r.size >= ?')
|
||||
bindings.push(params.minSize)
|
||||
}
|
||||
if (params.maxSize !== undefined) {
|
||||
where.push('r.size <= ?')
|
||||
bindings.push(params.maxSize)
|
||||
}
|
||||
// File types from file names and archive listings (migration …130), via their GIN index.
|
||||
const extensions = (params.extensions ?? []).map((e) => e.toLowerCase().replace(/^\./, '')).filter((e) => /^[a-z0-9]{1,8}$/.test(e)).slice(0, 8)
|
||||
if (extensions.length > 0) {
|
||||
where.push('r.extensions && ?::text[]')
|
||||
bindings.push(extensions)
|
||||
}
|
||||
if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
|
||||
if (params.hideContainers) where.push('r.container IS NULL')
|
||||
// par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
|
||||
if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
|
||||
// Kept current by triggers (migration …860).
|
||||
if (params.complete) where.push('r.complete')
|
||||
if (params.source) {
|
||||
where.push('r.source = ?')
|
||||
bindings.push(params.source)
|
||||
}
|
||||
if (params.poster) {
|
||||
where.push('r.poster ILIKE ?')
|
||||
bindings.push(pattern(params.poster))
|
||||
}
|
||||
if (params.categories?.length) {
|
||||
where.push('r.category = ANY(?)')
|
||||
bindings.push(params.categories)
|
||||
}
|
||||
if (params.notIncomplete) {
|
||||
where.push(`(r.complete OR (r.file_total IS NULL AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)))`)
|
||||
}
|
||||
if (params.maxAgeDays) {
|
||||
where.push(`r.posted_at >= now() - make_interval(days => ?)`)
|
||||
bindings.push(params.maxAgeDays)
|
||||
}
|
||||
if (params.season !== undefined) {
|
||||
const s = params.season
|
||||
const e = params.episode
|
||||
where.push('r.name ~* ?')
|
||||
bindings.push(
|
||||
e !== undefined
|
||||
? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)`
|
||||
: `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`
|
||||
)
|
||||
}
|
||||
|
||||
const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
|
||||
const page = Math.max(params.page ?? 1, 1)
|
||||
const offset = params.offset ?? (page - 1) * perPage
|
||||
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
||||
const whereSql = where.join(' AND ')
|
||||
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
||||
|
||||
const [{ rows }, { rows: countRows }] = await Promise.all([
|
||||
db.rawQuery(
|
||||
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.file_count, r.category, r.container,
|
||||
r.size, r.encrypted, is_readable_name(r.name) AS readable,
|
||||
(SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id) AS listed_files,
|
||||
(SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id AND EXISTS (
|
||||
SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
|
||||
jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
|
||||
r.complete, ${PAR2_ONLY_SQL} AS par2_only, r.password IS NOT NULL AS has_password,
|
||||
(SELECT s.description FROM spots s WHERE s.release_id = r.id AND s.description IS NOT NULL LIMIT 1) AS spot_description
|
||||
FROM releases r WHERE ${whereSql}
|
||||
ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
[...bindings, perPage, offset]
|
||||
),
|
||||
db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
|
||||
])
|
||||
|
||||
return {
|
||||
page,
|
||||
perPage,
|
||||
offset,
|
||||
total: Math.min(Number(countRows[0].total), countLimit),
|
||||
/** More than `total` match (the count stops at `countLimit`). */
|
||||
totalCapped: Number(countRows[0].total) > countLimit,
|
||||
releases: rows.map((r: any) => ({
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
source: r.source,
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
postedAt: r.posted_at,
|
||||
fileTotal: r.file_total,
|
||||
fileCount: r.file_count,
|
||||
size: Number(r.size),
|
||||
encrypted: r.encrypted,
|
||||
readable: r.readable,
|
||||
listedFiles: Number(r.listed_files),
|
||||
listedPresent: Number(r.listed_present),
|
||||
archiveEntries: Number(r.archive_entries),
|
||||
complete: r.complete,
|
||||
category: r.category,
|
||||
container: r.container,
|
||||
par2Only: r.par2_only,
|
||||
listingOnly: r.source === 'spot' && Number(r.file_count) === 0,
|
||||
hasPassword: r.has_password,
|
||||
passwordState: passwordState(r.encrypted, r.has_password),
|
||||
spotDescription: r.spot_description,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
async release(id: number) {
|
||||
const release = await db.from('releases').where('id', id).first()
|
||||
if (!release) return null
|
||||
|
||||
const [files, listed, pending] = await Promise.all([
|
||||
db
|
||||
.from('files as f')
|
||||
.leftJoin('binaries as b', 'b.file_id', 'f.id')
|
||||
.where('f.release_id', id)
|
||||
.orderByRaw('coalesce(f.real_name, f.filename)')
|
||||
.select('f.id', 'f.filename', 'f.real_name', 'f.size', 'f.parts', 'f.date', 'f.poster', 'b.size as exact_size'),
|
||||
db.rawQuery(
|
||||
`SELECT rf.name, rf.size,
|
||||
EXISTS (SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = rf.release_id AND (f.real_name = rf.name OR b.size = rf.size)) AS present
|
||||
FROM release_files rf WHERE rf.release_id = ? ORDER BY rf.name`,
|
||||
[id]
|
||||
),
|
||||
db.rawQuery(
|
||||
`SELECT count(*) AS binaries, coalesce(sum(b.total_parts), 0) AS parts_total,
|
||||
coalesce(sum((SELECT count(*) FROM parts p WHERE p.binary_id = b.id)), 0) AS parts_have
|
||||
FROM binaries b WHERE b.release_id = ? AND b.completed_at IS NULL AND b.conflicted_at IS NULL`,
|
||||
[id]
|
||||
),
|
||||
])
|
||||
|
||||
const spotRow = await db.from('spots').where('release_id', id).whereNotIn('status', ['removed', 'dummy']).orderBy('posted_at', 'desc').first()
|
||||
const spot = spotRow ? await SpotService.withDetails(spotRow) : null
|
||||
return {
|
||||
id: Number(release.id),
|
||||
name: release.name,
|
||||
source: release.source,
|
||||
spot: spot
|
||||
? {
|
||||
messageId: spot.message_id,
|
||||
title: spot.title,
|
||||
tag: spot.tag,
|
||||
poster: spot.poster,
|
||||
postedAt: spot.posted_at,
|
||||
website: spot.website,
|
||||
description: spot.description,
|
||||
image: spot.image?.segments
|
||||
? { url: `/api/v1/releases/${id}/image`, width: spot.image.width ?? null, height: spot.image.height ?? null }
|
||||
: spot.image?.url
|
||||
? { external: spot.image.url }
|
||||
: null,
|
||||
categories: spotCategoryLabels(spot.category, spot.subcats),
|
||||
}
|
||||
: null,
|
||||
password: release.password ?? null,
|
||||
passwordSource: release.password_source ?? null,
|
||||
passwordState: passwordState(release.encrypted, release.password !== null),
|
||||
listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
|
||||
poster: release.poster,
|
||||
groups: release.groups,
|
||||
postedAt: release.posted_at,
|
||||
fileTotal: release.file_total,
|
||||
fileCount: release.file_count,
|
||||
size: Number(release.size),
|
||||
encrypted: release.encrypted,
|
||||
container: release.container,
|
||||
contents: release.contents ?? [],
|
||||
files: files.map((f) => ({
|
||||
id: Number(f.id),
|
||||
name: f.real_name ?? f.filename,
|
||||
postedName: f.filename,
|
||||
realName: f.real_name,
|
||||
size: Number(f.exact_size ?? f.size),
|
||||
parts: f.parts,
|
||||
postedAt: f.date ? new Date(Number(f.date)) : null,
|
||||
poster: f.poster,
|
||||
})),
|
||||
par2Files: listed.rows.map((r: any) => ({ name: r.name, size: Number(r.size), present: r.present })),
|
||||
pending: {
|
||||
binaries: Number(pending.rows[0].binaries),
|
||||
partsHave: Number(pending.rows[0].parts_have),
|
||||
partsTotal: Number(pending.rows[0].parts_total),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** The image of the spot behind a release: bytes and a media type sniffed from them, or null. */
|
||||
async spotImage(id: number): Promise<{ data: Buffer; type: string } | null> {
|
||||
const spot = await db.from('spots').where('release_id', id).whereNotNull('image').orderBy('posted_at', 'desc').first()
|
||||
if (!spot) return null
|
||||
const data = await SpotService.image(spot)
|
||||
if (!data) return null
|
||||
const type =
|
||||
data[0] === 0xff && data[1] === 0xd8 ? 'image/jpeg'
|
||||
: data.subarray(0, 4).toString('latin1') === '\x89PNG' ? 'image/png'
|
||||
: data.subarray(0, 3).toString('latin1') === 'GIF' ? 'image/gif'
|
||||
: data.subarray(0, 2).toString('latin1') === 'BM' ? 'image/bmp'
|
||||
: data.subarray(8, 12).toString('latin1') === 'WEBP' ? 'image/webp'
|
||||
: null
|
||||
return type ? { data, type } : null
|
||||
}
|
||||
|
||||
async nzb(id: number): Promise<{ name: string; xml: string } | null> {
|
||||
let release = await db.from('releases').where('id', id).first()
|
||||
if (!release) return null
|
||||
if (release.source === 'spot' && Number(release.file_count) === 0) {
|
||||
// A Spotnet listing: read its NZB from Usenet now (the import may join known files elsewhere).
|
||||
const imported = await SpotService.importListing(id)
|
||||
if (!imported) return null
|
||||
id = imported
|
||||
release = await db.from('releases').where('id', id).first()
|
||||
}
|
||||
const files = await db
|
||||
.from('files')
|
||||
.where('release_id', id)
|
||||
.orderByRaw('coalesce(real_name, filename)')
|
||||
.select('filename', 'real_name', 'poster', 'date', 'groups', 'segments')
|
||||
const xml = buildNzb(
|
||||
files.map((f) => ({
|
||||
name: f.real_name ?? f.filename,
|
||||
poster: f.poster,
|
||||
date: Number(f.date),
|
||||
groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
|
||||
segments: segmentsOf(f),
|
||||
})),
|
||||
{ title: release.name, password: release.password ?? undefined }
|
||||
)
|
||||
return { name: release.name, xml }
|
||||
}
|
||||
|
||||
/**
|
||||
* Totals for the web UI header. They count millions of rows (minutes with
|
||||
* the backfill running), so they are computed at most once per
|
||||
* STATS_TTL_MS and never twice at a time; callers in between share it.
|
||||
*/
|
||||
async stats() {
|
||||
if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
|
||||
this.statsRefresh ??= this.computeStats()
|
||||
.then((value) => {
|
||||
this.statsCache = { at: Date.now(), value }
|
||||
})
|
||||
// A failed background refresh keeps the previous totals; only a caller that waits sees the error.
|
||||
.catch((error) => {
|
||||
if (!this.statsCache) throw error
|
||||
})
|
||||
.finally(() => {
|
||||
this.statsRefresh = undefined
|
||||
})
|
||||
// Only the very first caller waits; later ones get the previous totals meanwhile.
|
||||
if (!this.statsCache) await this.statsRefresh
|
||||
}
|
||||
return this.statsCache!.value
|
||||
}
|
||||
|
||||
private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
|
||||
private statsRefresh?: Promise<void>
|
||||
|
||||
/**
|
||||
* Estimates, in milliseconds instead of minutes (CONCEPT.md D33): counts
|
||||
* from Postgres's statistics (visible releases = the entries of an index
|
||||
* that covers exactly them), ratios and sizes from a 0.5% sample of the
|
||||
* releases table. Within a few percent once autovacuum has analysed.
|
||||
*/
|
||||
private async computeStats() {
|
||||
const { rows } = await db.rawQuery(`
|
||||
WITH sample AS (
|
||||
SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
|
||||
FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
|
||||
SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
|
||||
-- files is partitioned (D39): its partitions hold the counts.
|
||||
(SELECT sum(c.reltuples) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass) AS files,
|
||||
(SELECT count(*) FROM sample) AS sampled,
|
||||
(SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
|
||||
-- Indexed data only: a spot listing's size is what the spot announces.
|
||||
(SELECT coalesce(sum(size), 0) FROM sample WHERE file_count > 0) AS sampled_size,
|
||||
(SELECT count(*) FROM sample WHERE encrypted) AS sampled_encrypted,
|
||||
(SELECT max(updated_at) FROM groups) AS last_indexed_at,
|
||||
(SELECT json_agg(json_build_object('source', source, 'n', n) ORDER BY n DESC)
|
||||
FROM (SELECT source, count(*) AS n FROM sample GROUP BY 1) s) AS sources`)
|
||||
const r = rows[0]
|
||||
const releases = Math.max(0, Number(r.releases))
|
||||
const sampled = Number(r.sampled)
|
||||
// Sample counts scaled to all visible releases.
|
||||
const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
|
||||
return {
|
||||
approximate: true,
|
||||
releases,
|
||||
readableReleases: scale(r.sampled_readable),
|
||||
files: Math.max(0, Number(r.files)),
|
||||
size: scale(r.sampled_size),
|
||||
encryptedReleases: scale(r.sampled_encrypted),
|
||||
lastIndexedAt: r.last_indexed_at,
|
||||
sources: (r.sources ?? []).map((s: any) => ({ source: s.source, releases: scale(s.n) })),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new SearchService()
|
||||
@@ -1,132 +0,0 @@
|
||||
import nntpConfig from '#config/nntp'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import { XoverClient } from '#lib/nntp/xover_client'
|
||||
|
||||
interface Slot {
|
||||
client: XoverClient
|
||||
group?: string
|
||||
usedAt: number
|
||||
}
|
||||
|
||||
/**
|
||||
* A pooled connection unused this long is replaced: the news server drops
|
||||
* idle connections without telling, and a request on one then hangs until
|
||||
* the inactivity timeout (LOG.md 2026-10-05 (11)).
|
||||
*/
|
||||
const IDLE_MS = 60 * 1000
|
||||
|
||||
/**
|
||||
* NNTP connections in the seed API for relay nodes (CONCEPT.md D38): the
|
||||
* indexer fetches overview chunks and passes the gzip stream on unopened, so
|
||||
* nodes need no news server account and every connection comes from this
|
||||
* machine's IP. SEED_NNTP_CONNECTIONS (0: relay off).
|
||||
*/
|
||||
class SeedFetchPool {
|
||||
private idle: Slot[] = []
|
||||
private open = 0
|
||||
private waiters: (() => void)[] = []
|
||||
|
||||
get size() {
|
||||
return indexerConfig.seed.nntpConnections
|
||||
}
|
||||
|
||||
/**
|
||||
* Pooled connections fetch compressed unless SEED_NNTP_COMPRESS=false. A failure is retried once without compression, on a connection of its
|
||||
* own: either the pooled connection was dead (dropped while idle), or the
|
||||
* server stalled mid-reply in compressed mode. Eweka does that for some
|
||||
* chunks: it stops after most of the stream and never sends the rest
|
||||
* (a.b.ath 49701720000: 19,672 of 20,000 overviews, then silence), while
|
||||
* the same chunk uncompressed comes in 0.3 s (LOG.md 2026-10-05 (11)).
|
||||
*/
|
||||
async fetch(group: string, from: number, to: number): Promise<Buffer> {
|
||||
try {
|
||||
return await this.fetchOnce(group, from, to)
|
||||
} catch {
|
||||
return this.fetchPlain(group, from, to)
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchPlain(group: string, from: number, to: number): Promise<Buffer> {
|
||||
const client = await this.connect(false)
|
||||
try {
|
||||
await client.group(group)
|
||||
return await client.xoverRaw(from, to)
|
||||
} finally {
|
||||
client.close()
|
||||
}
|
||||
}
|
||||
|
||||
private connect(compress: boolean) {
|
||||
return XoverClient.connect({
|
||||
host: nntpConfig.host,
|
||||
port: nntpConfig.port,
|
||||
secure: nntpConfig.secure,
|
||||
servername: nntpConfig.tlsServername,
|
||||
user: nntpConfig.user,
|
||||
password: nntpConfig.password?.release(),
|
||||
timeoutMs: indexerConfig.seed.nntpTimeoutMs,
|
||||
compress,
|
||||
})
|
||||
}
|
||||
|
||||
private async fetchOnce(group: string, from: number, to: number): Promise<Buffer> {
|
||||
const slot = await this.acquire()
|
||||
let ok = false
|
||||
try {
|
||||
if (slot.group !== group) {
|
||||
await slot.client.group(group)
|
||||
slot.group = group
|
||||
}
|
||||
const raw = await slot.client.xoverRaw(from, to)
|
||||
ok = true
|
||||
return raw
|
||||
} finally {
|
||||
this.release(slot, ok)
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes the idle connections (tests, shutdown); busy ones close when released. */
|
||||
closeIdle() {
|
||||
for (const slot of this.idle.splice(0)) {
|
||||
slot.client.close()
|
||||
this.open--
|
||||
}
|
||||
}
|
||||
|
||||
private async acquire(): Promise<Slot> {
|
||||
while (true) {
|
||||
const slot = this.idle.pop()
|
||||
if (slot) {
|
||||
if (Date.now() - slot.usedAt < IDLE_MS) return slot
|
||||
slot.client.close()
|
||||
this.open--
|
||||
continue
|
||||
}
|
||||
if (this.open < this.size) {
|
||||
this.open++
|
||||
try {
|
||||
return { client: await this.connect(indexerConfig.seed.nntpCompress), usedAt: Date.now() }
|
||||
} catch (error) {
|
||||
this.open--
|
||||
this.waiters.shift()?.()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
await new Promise<void>((resolve) => this.waiters.push(resolve))
|
||||
}
|
||||
}
|
||||
|
||||
private release(slot: Slot, ok: boolean) {
|
||||
if (ok) {
|
||||
slot.usedAt = Date.now()
|
||||
this.idle.push(slot)
|
||||
} else {
|
||||
// A failed command can leave the connection mid-reply: start afresh.
|
||||
slot.client.close()
|
||||
this.open--
|
||||
}
|
||||
this.waiters.shift()?.()
|
||||
}
|
||||
}
|
||||
|
||||
export default new SeedFetchPool()
|
||||
@@ -1,234 +0,0 @@
|
||||
import { createHash, randomBytes } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { cut, uncovered } from '#lib/seed/plan'
|
||||
import NntpService from '#services/NntpService'
|
||||
import SeedService from '#services/SeedService'
|
||||
import BackfillService from '#services/BackfillService'
|
||||
import SeedFetchPool from '#services/SeedFetchPool'
|
||||
import { articleAtDate } from '#lib/nntp/date_search'
|
||||
import { normalizeSeedFile, type SeedBatch } from '#lib/seed/wire'
|
||||
|
||||
/** A lease without a batch for this long is given to another node; the range resumes at next_id. */
|
||||
export const LEASE_TTL_SECONDS = 15 * 60
|
||||
/** Leases per range before it counts as failed (a range that keeps breaking nodes). */
|
||||
const MAX_ATTEMPTS = 5
|
||||
/** Articles per XOVER probe when looking for an article by date. */
|
||||
const PROBE = 50
|
||||
|
||||
export interface SeedLease {
|
||||
id: number
|
||||
group: string
|
||||
/** Resume point: first article not reported yet. */
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
export class LeaseLost extends Error {}
|
||||
|
||||
const hash = (token: string) => createHash('sha256').update(token).digest('hex')
|
||||
|
||||
/**
|
||||
* The seed orchestrator (CONCEPT.md D38): seeding nodes lease article
|
||||
* ranges over HTTPS, report what they aggregated in batches (D37), and
|
||||
* finish or give back the range. Nodes only make outgoing requests; a lease
|
||||
* that goes quiet for LEASE_TTL_SECONDS passes to the next node that asks.
|
||||
*/
|
||||
class SeedOrchestrator {
|
||||
/** Creates a node, or gives an existing one a new token. Returns the token (stored hashed only). */
|
||||
async issueToken(name: string): Promise<string> {
|
||||
const token = `sn_${randomBytes(24).toString('base64url')}`
|
||||
await db
|
||||
.table('seed_nodes')
|
||||
.insert({ name, token_hash: hash(token), disabled: false })
|
||||
.onConflict('name')
|
||||
.merge({ token_hash: hash(token), disabled: false })
|
||||
return token
|
||||
}
|
||||
|
||||
async nodeForToken(token: string): Promise<{ id: number; name: string } | null> {
|
||||
const node = await db.from('seed_nodes').where('token_hash', hash(token)).where('disabled', false).first()
|
||||
if (!node) return null
|
||||
if (!node.last_seen_at || Date.now() - new Date(node.last_seen_at).getTime() > 60_000) {
|
||||
await db.from('seed_nodes').where('id', node.id).update({ last_seen_at: new Date() })
|
||||
}
|
||||
return { id: Number(node.id), name: node.name }
|
||||
}
|
||||
|
||||
/**
|
||||
* The next range for this node, oldest plan first; a quiet lease is taken
|
||||
* over. Nothing while the database is over the backfill's disk limits: the
|
||||
* node asks again a minute later.
|
||||
*/
|
||||
async lease(nodeId: number): Promise<SeedLease | null> {
|
||||
if ((await BackfillService.room()) !== true) return null
|
||||
// Fully reported but never finished (the node went away at the very end): nothing left to fetch.
|
||||
await db.rawQuery(
|
||||
`UPDATE seed_ranges SET status = 'done', done_at = now(), node_id = NULL
|
||||
WHERE next_id > end_id AND (status = 'queued' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))`,
|
||||
[LEASE_TTL_SECONDS]
|
||||
)
|
||||
// Ranges that broke node after node stop being handed out.
|
||||
await db.rawQuery(
|
||||
`UPDATE seed_ranges SET status = 'failed', error = coalesce(error, 'lease expired too often')
|
||||
WHERE status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?) AND attempts >= ?`,
|
||||
[LEASE_TTL_SECONDS, MAX_ATTEMPTS]
|
||||
)
|
||||
// Priorities take turns: while ranges of a higher priority are leased for less than half of
|
||||
// the work in progress they go first, otherwise the rest does (a 2026 gap fetches ~5x slower
|
||||
// than older history; LOG.md 2026-10-06 (10)).
|
||||
const { rows: mix } = await db.rawQuery(
|
||||
`SELECT count(*) FILTER (WHERE priority > 0) AS high, count(*) AS all FROM seed_ranges
|
||||
WHERE status = 'leased' AND heartbeat_at >= now() - make_interval(secs => ?)`,
|
||||
[LEASE_TTL_SECONDS]
|
||||
)
|
||||
const order = Number(mix[0].high) * 2 < Number(mix[0].all) + 1 ? 'priority DESC, id' : 'priority ASC, id'
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE seed_ranges SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1
|
||||
WHERE id = (
|
||||
SELECT id FROM seed_ranges
|
||||
WHERE status = 'queued' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?))
|
||||
ORDER BY ${order} LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
RETURNING id, group_name, next_id, end_id`,
|
||||
[nodeId, LEASE_TTL_SECONDS]
|
||||
)
|
||||
const row = rows[0]
|
||||
return row ? { id: Number(row.id), group: row.group_name, start: Number(row.next_id), end: Number(row.end_id) } : null
|
||||
}
|
||||
|
||||
/** Stores a node's batch and moves the range's resume point; LeaseLost when the range is no longer this node's. */
|
||||
async batch(nodeId: number, rangeId: number, batch: SeedBatch) {
|
||||
const range = await this.leased(nodeId, rangeId)
|
||||
const stored = await SeedService.store(batch.files.map(normalizeSeedFile))
|
||||
await SeedService.storeFragments(batch.fragments, { start: Number(range.start_id), end: Number(range.end_id) })
|
||||
await db.rawQuery(
|
||||
`UPDATE seed_ranges SET next_id = greatest(next_id, least(?, end_id + 1)), heartbeat_at = now(),
|
||||
headers = headers + ?, files = files + ?, fragments = fragments + ?
|
||||
WHERE id = ?`,
|
||||
[batch.next, batch.headers, batch.files.length, batch.fragments.length, rangeId]
|
||||
)
|
||||
return stored
|
||||
}
|
||||
|
||||
/**
|
||||
* Relay: overviews of `from`..`to` of a leased range, fetched here and handed
|
||||
* on as the gzip stream the news server sent. Keeps the lease alive.
|
||||
*/
|
||||
async chunk(nodeId: number, rangeId: number, from: number, to: number): Promise<Buffer> {
|
||||
const range = await this.leased(nodeId, rangeId)
|
||||
const start = Math.max(from, Number(range.start_id))
|
||||
const end = Math.min(to, Number(range.end_id), start + 49_999)
|
||||
if (end < start) throw new RangeError(`Articles ${from}–${to} are outside range ${rangeId}`)
|
||||
await db.from('seed_ranges').where('id', rangeId).update({ heartbeat_at: new Date() })
|
||||
return SeedFetchPool.fetch(range.group_name, start, end)
|
||||
}
|
||||
|
||||
async finish(nodeId: number, rangeId: number) {
|
||||
await this.leased(nodeId, rangeId)
|
||||
await db.from('seed_ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date(), error: null })
|
||||
}
|
||||
|
||||
/**
|
||||
* A node gives a range back (it failed, or is shutting down); it's queued
|
||||
* again unless it failed too often. `uncounted`: the node never started on
|
||||
* it (couldn't reach its news server), so the lease doesn't count as an attempt.
|
||||
*/
|
||||
async release(nodeId: number, rangeId: number, error?: string, uncounted = false) {
|
||||
const range = await this.leased(nodeId, rangeId)
|
||||
const attempts = uncounted ? Math.max(0, range.attempts - 1) : range.attempts
|
||||
await db
|
||||
.from('seed_ranges')
|
||||
.where('id', rangeId)
|
||||
.update({ status: attempts >= MAX_ATTEMPTS ? 'failed' : 'queued', node_id: null, attempts, error: error?.slice(0, 2000) ?? null })
|
||||
}
|
||||
|
||||
private async leased(nodeId: number, rangeId: number) {
|
||||
const range = await db.from('seed_ranges').where('id', rangeId).first()
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
|
||||
return range
|
||||
}
|
||||
|
||||
/**
|
||||
* Plans ranges for groups: from below what live indexing and the backfill
|
||||
* cover (or below earlier seed ranges) down to `days` back, or to the
|
||||
* server's oldest article. A group's backfill stops where its seed plan
|
||||
* begins. Ranges are interleaved across groups, newest first.
|
||||
*/
|
||||
async plan(
|
||||
groups: string[],
|
||||
options: { days?: number; size: number; priority?: number }
|
||||
): Promise<{ group: string; from: number; to: number; ranges: number; gaps: number }[]> {
|
||||
const perGroup: { group: string; ranges: [number, number][] }[] = []
|
||||
const summary: { group: string; from: number; to: number; ranges: number; gaps: number }[] = []
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
for (const name of groups) {
|
||||
const group = await db.from('groups').where('name', name).first()
|
||||
if (!group) throw new Error(`Unknown group ${name}`)
|
||||
const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, name), `GROUP ${name}`)
|
||||
const dateOf = async (n: bigint) => {
|
||||
const { overviews } = await NntpService.timed<any>(conn.xover(n, n + BigInt(PROBE)), `XOVER ${name} ${n}`)
|
||||
const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
|
||||
const date = header ? Date.parse(header.date) : Number.NaN
|
||||
return Number.isNaN(date) ? null : date
|
||||
}
|
||||
// Top: where live indexing began, or as far down as the backfill has come.
|
||||
let top = Number(await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf))
|
||||
if (group.backfill_next_id !== null) top = Math.min(top, Number(group.backfill_next_id))
|
||||
const backfilling = group.backfill_next_id !== null && group.backfill_until_id !== null && Number(group.backfill_next_id) > Number(group.backfill_until_id)
|
||||
const bottom = options.days
|
||||
? Number(await articleAtDate(first, last, Date.now() - options.days * 24 * 60 * 60 * 1000, dateOf))
|
||||
: Number(first)
|
||||
|
||||
// Everything below the top that no earlier range covers (in any state). Planning below
|
||||
// the lowest earlier range left a 13-month gap in a.b.ath (LOG.md 2026-10-06 (7)).
|
||||
const planned = await db.from('seed_ranges').where('group_name', name).select('start_id', 'end_id')
|
||||
const gaps = uncovered(bottom, top - 1, planned.map((r) => [Number(r.start_id), Number(r.end_id)] as [number, number]))
|
||||
const ranges = cut(gaps, options.size)
|
||||
perGroup.push({ group: name, ranges })
|
||||
summary.push({ group: name, from: bottom, to: top - 1, ranges: ranges.length, gaps: gaps.length })
|
||||
// The seed plan takes over the rest of this group's backfill.
|
||||
if (backfilling && ranges.length > 0) {
|
||||
await db.from('groups').where('id', group.id).update({ backfill_until_id: group.backfill_next_id })
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
|
||||
// Round-robin over groups, each newest first, so every group fills in from the present backwards.
|
||||
const rows: Record<string, unknown>[] = []
|
||||
for (let i = 0; perGroup.some((g) => i < g.ranges.length); i++) {
|
||||
for (const g of perGroup) {
|
||||
const range = g.ranges[i]
|
||||
if (range) rows.push({ group_name: g.group, start_id: range[0], end_id: range[1], next_id: range[0], priority: options.priority ?? 0 })
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < rows.length; i += 5000) await db.table('seed_ranges').multiInsert(rows.slice(i, i + 5000))
|
||||
return summary
|
||||
}
|
||||
|
||||
async status() {
|
||||
const { rows: ranges } = await db.rawQuery(
|
||||
`SELECT status, count(*)::int AS ranges, sum(end_id - start_id + 1)::bigint AS articles,
|
||||
sum(next_id - start_id)::bigint AS reached, sum(headers)::bigint AS headers, sum(files)::bigint AS files
|
||||
FROM seed_ranges GROUP BY status ORDER BY status`
|
||||
)
|
||||
const { rows: nodes } = await db.rawQuery(
|
||||
`SELECT n.name, n.last_seen_at, n.disabled,
|
||||
(SELECT count(*)::int FROM seed_ranges r WHERE r.node_id = n.id AND r.status = 'leased') AS leases,
|
||||
(SELECT coalesce(sum(headers), 0)::bigint FROM seed_ranges r WHERE r.node_id = n.id) AS headers
|
||||
FROM seed_nodes n ORDER BY n.name`
|
||||
)
|
||||
const { rows: recent } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(headers), 0)::bigint AS headers FROM seed_ranges WHERE done_at > now() - interval '1 hour'`
|
||||
)
|
||||
const room = await BackfillService.room()
|
||||
return { ranges, nodes, headersDoneLastHour: Number(recent[0].headers), paused: room === true ? null : room }
|
||||
}
|
||||
}
|
||||
|
||||
export default new SeedOrchestrator()
|
||||
@@ -1,152 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import IngestService from '#services/IngestService'
|
||||
import FileStore, { insertLimits } from '#services/FileStore'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import { fromFragment, type SeedFile, type SeedFragment } from '#lib/seed/wire'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
|
||||
/** Articles from a range's edge within which a file may continue in the neighbouring range. */
|
||||
const EDGE_ARTICLES = 50_000
|
||||
|
||||
export interface SeedStoreResult {
|
||||
files: number
|
||||
/** Files that existed already (same file_key, e.g. from live indexing or a crosspost). */
|
||||
existing: number
|
||||
releases: number
|
||||
inspections: number
|
||||
/** Random-named files of garbage signatures or dropped releases, not stored (D42). */
|
||||
dropped: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores what seeding nodes found (CONCEPT.md D37): per complete file its
|
||||
* release, the file with its segment list, and one completed binary, as
|
||||
* worker:file would have left it, so par2 naming, inspection and GC treat
|
||||
* it like any other file; fragments take the regular ingest path. A batch
|
||||
* is a few statements, not one per article.
|
||||
*/
|
||||
class SeedService {
|
||||
async store(batch: SeedFile[]): Promise<SeedStoreResult> {
|
||||
const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0, dropped: 0 }
|
||||
// A repost can complete the same file twice in one batch; ON CONFLICT DO UPDATE takes each row once.
|
||||
const unique = [...new Map(batch.map((file) => [file.fileKey, file])).values()]
|
||||
// Garbage isn't stored at all: no release, file or binary (CONCEPT.md D42).
|
||||
const { kept: files, dropped } = await GarbageFilter.keep(unique, (file) => ({
|
||||
name: file.name,
|
||||
messageId: file.segments[0]?.[1],
|
||||
groups: file.groups,
|
||||
releaseKey: file.release?.key ?? file.fileKey,
|
||||
}))
|
||||
result.dropped = dropped
|
||||
if (files.length === 0) return result
|
||||
const knex = db.connection().getWriteClient()
|
||||
const now = new Date()
|
||||
const postedOf = (file: SeedFile) => (file.postedAt ? new Date(file.postedAt) : undefined)
|
||||
|
||||
const releaseIds = await IngestService.upsertReleases(
|
||||
knex,
|
||||
files.map((file) => ({ release: file.release, poster: file.poster, groups: file.groups, postedAt: postedOf(file) })),
|
||||
now
|
||||
)
|
||||
result.releases = releaseIds.size
|
||||
|
||||
const rows = files.map((file) => ({
|
||||
file_key: file.fileKey,
|
||||
filename: file.name,
|
||||
poster: file.poster,
|
||||
date: (postedOf(file) ?? now).getTime(),
|
||||
parts: file.segments.length,
|
||||
segments: file.segments.map(([number, id, size]) => ({ number, id, size })),
|
||||
groups: JSON.stringify(file.groups),
|
||||
release_id: file.release ? (releaseIds.get(file.release.key) ?? null) : null,
|
||||
size: file.segments.reduce((sum, [, , size]) => sum + size, 0),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}))
|
||||
const inserted = await FileStore.insert(rows)
|
||||
result.files = inserted.length
|
||||
result.existing = rows.length - inserted.length
|
||||
|
||||
// File ids for the binaries below (files queued for inspection only): new ones from the insert,
|
||||
// existing ones looked up. Looking up every key of the batch probed all files partitions again,
|
||||
// 15% of Postgres' active time (LOG.md 2026-10-06 (12)).
|
||||
const fileIds = new Map<string, number>()
|
||||
for (const row of inserted) if (row.file_key) fileIds.set(row.file_key, row.id)
|
||||
const missing = files.filter((file) => !fileIds.has(file.fileKey) && ContentService.isCandidate(file.name)).map((file) => file.fileKey)
|
||||
for (let i = 0; i < missing.length; i += 5000) {
|
||||
for (const row of await db.from('files').whereIn('file_key', missing.slice(i, i + 5000)).select('id', 'file_key')) {
|
||||
fileIds.set(row.file_key, Number(row.id))
|
||||
}
|
||||
}
|
||||
// A completed binary, as worker:file would have left behind, only for files queued for content
|
||||
// inspection: par2 naming (D16) needs the fingerprint inspection stores on it. For the others
|
||||
// (~85% in a.b.ath) it was only GC churn: 1.6M rows an hour inserted and deleted (LOG.md 2026-10-06 (11)).
|
||||
const insertedKeys = new Set(inserted.map((row) => row.file_key))
|
||||
const binaries = files.flatMap((file, i) => {
|
||||
if (!ContentService.isCandidate(file.name)) return []
|
||||
return [{
|
||||
file_key: file.fileKey,
|
||||
poster: file.poster,
|
||||
subject: file.subject,
|
||||
name: file.name,
|
||||
total_parts: file.totalParts,
|
||||
groups: rows[i].groups,
|
||||
posted_at: postedOf(file) ?? null,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
completed_at: now,
|
||||
release_id: rows[i].release_id,
|
||||
file_id: fileIds.get(file.fileKey) ?? null,
|
||||
// New files are queued for inspection below; an existing file was queued when it was added.
|
||||
awaiting_inspection: insertedKeys.has(file.fileKey) ? true : null,
|
||||
}]
|
||||
})
|
||||
binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
|
||||
for (let i = 0; i < binaries.length; i += insertLimits.rows) {
|
||||
await knex('binaries')
|
||||
.insert(binaries.slice(i, i + insertLimits.rows))
|
||||
.onConflict('file_key')
|
||||
.merge({
|
||||
file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any,
|
||||
awaiting_inspection: knex.raw('coalesce(binaries.awaiting_inspection, excluded.awaiting_inspection)') as any,
|
||||
})
|
||||
}
|
||||
|
||||
// As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
|
||||
const rowByKey = new Map(rows.map((row) => [row.file_key, row]))
|
||||
const added = inserted.flatMap((file) => {
|
||||
const row = file.file_key ? rowByKey.get(file.file_key) : undefined
|
||||
return row ? [{ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }] : []
|
||||
})
|
||||
await SfvService.claimAll(added)
|
||||
const candidates = added
|
||||
.filter((file) => ContentService.isCandidate(file.name))
|
||||
.map((file) => file.id)
|
||||
if (candidates.length > 0) await InspectionQueue.add(candidates)
|
||||
result.inspections = candidates.length
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Segments of files that didn't complete on a node. Only files near the
|
||||
* range's edges can continue in the neighbouring range (worked on by some
|
||||
* node, maybe hours later): those go straight into binaries and parts
|
||||
* (staging would drop them after 30 minutes). The rest are files with
|
||||
* articles missing for good, and are dropped: staging them filled 22 GB
|
||||
* of churn on the NVMe (LOG.md 2026-10-05 (10)). Returns the segments kept.
|
||||
*/
|
||||
async storeFragments(fragments: SeedFragment[], range: { start: number; end: number }): Promise<number> {
|
||||
const nearEdge = new Set<string>()
|
||||
for (const fragment of fragments) {
|
||||
const n = Number(fragment.articleNumber)
|
||||
if (Number.isFinite(n) && (n < range.start + EDGE_ARTICLES || n > range.end - EDGE_ARTICLES)) nearEdge.add(fragment.fileKey)
|
||||
}
|
||||
const kept = fragments.filter((fragment) => nearEdge.has(fragment.fileKey))
|
||||
if (kept.length > 0) await IngestService.ingest(kept.map(fromFragment))
|
||||
return kept.length
|
||||
}
|
||||
}
|
||||
|
||||
export default new SeedService()
|
||||
@@ -1,36 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { sevenZipVolume } from '#lib/sevenzip/index'
|
||||
|
||||
/**
|
||||
* The database side of 7z listings (CONCEPT.md D19): which volumes make up
|
||||
* an archive. Listing itself (first segment, tail, bsdtar) is
|
||||
* lib/sevenzip/list.ts, run by inspection here or on a node.
|
||||
*/
|
||||
class SevenZipService {
|
||||
/**
|
||||
* The volumes of the archive `file` belongs to, in order, when all of them
|
||||
* up to the highest number seen are there; null otherwise. An unsplit (or
|
||||
* unnamed) archive is its own only volume.
|
||||
*/
|
||||
async volumes(file: any): Promise<any[] | null> {
|
||||
const name = String(file.real_name ?? file.filename)
|
||||
const volume = sevenZipVolume(name)
|
||||
if (!volume) return [file]
|
||||
if (!file.release_id) return null
|
||||
const base = name.replace(/\.\d+$/, '').toLowerCase()
|
||||
const rows = await db
|
||||
.from('files')
|
||||
.where('release_id', file.release_id)
|
||||
.whereRaw(`lower(coalesce(real_name, filename)) LIKE ? ESCAPE '\\'`, [`${base.replace(/[\\%_]/g, '\\$&')}.%`])
|
||||
const byNumber = new Map<number, any>()
|
||||
for (const row of rows) {
|
||||
const n = sevenZipVolume(String(row.real_name ?? row.filename))
|
||||
if (n && !byNumber.has(n)) byNumber.set(n, row)
|
||||
}
|
||||
const max = Math.max(0, ...byNumber.keys())
|
||||
const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1))
|
||||
return max > 0 && ordered.every(Boolean) ? ordered : null
|
||||
}
|
||||
}
|
||||
|
||||
export default new SevenZipService()
|
||||
@@ -1,96 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import FileStore from '#services/FileStore'
|
||||
import { parseSfv, sfvReleaseName } from '#lib/content/index'
|
||||
|
||||
/** Releases made by name rules alone; an SFV's listing outranks them. */
|
||||
const WEAK_SOURCES = ['stem', 'weak-stem', 'subject']
|
||||
/** Files further apart than this from the SFV in posting time aren't taken. */
|
||||
const WINDOW_MS = 2 * 24 * 60 * 60 * 1000
|
||||
|
||||
/**
|
||||
* SFV grouping (CONCEPT.md D34). An uploader that posts every track of an
|
||||
* album as its own file (13.5k such releases from one poster on homelab)
|
||||
* still posts the album's SFV, which names them all.
|
||||
*/
|
||||
class SfvService {
|
||||
/**
|
||||
* Records an SFV file's listing on its release, moves the listed files that
|
||||
* sit in weak releases into it, and names the release after the album when
|
||||
* it's still named after a track or the SFV itself. Returns how many files
|
||||
* moved, or null when it isn't an SFV.
|
||||
*/
|
||||
async attach(sfvFile: any, text: string): Promise<number | null> {
|
||||
const entries = parseSfv(text)
|
||||
if (entries.length === 0 || !sfvFile.release_id) return null
|
||||
const releaseId = Number(sfvFile.release_id)
|
||||
const postedAt = sfvFile.date ? new Date(Number(sfvFile.date)) : null
|
||||
await db
|
||||
.knexQuery()
|
||||
.table('sfv_entries')
|
||||
.insert(entries.map((e) => ({ release_id: releaseId, name: e.name, crc: e.crc, poster: sfvFile.poster, posted_at: postedAt })))
|
||||
.onConflict(['release_id', 'name'])
|
||||
.ignore()
|
||||
await FileStore.setDetails(Number(sfvFile.id), { content_type: 'sfv' })
|
||||
|
||||
const candidates = await db
|
||||
.from('files as f')
|
||||
.join('releases as r', 'r.id', 'f.release_id')
|
||||
.whereIn(db.raw('coalesce(f.real_name, f.filename)') as any, entries.map((e) => e.name))
|
||||
.whereNot('f.release_id', releaseId)
|
||||
.whereIn('r.source', WEAK_SOURCES)
|
||||
.select('f.id', 'f.poster', 'f.date')
|
||||
const moving = candidates.filter((f) => this.sameUpload(f, sfvFile.poster, postedAt))
|
||||
await this.move(moving.map((f) => Number(f.id)), releaseId)
|
||||
|
||||
const release = await db.from('releases').where('id', releaseId).first()
|
||||
const album = sfvReleaseName(String(sfvFile.real_name ?? sfvFile.filename))
|
||||
if (release && WEAK_SOURCES.includes(release.source) && album !== release.name && /^\d{1,3}-/.test(release.name)) {
|
||||
await db.from('releases').where('id', releaseId).update({ name: album })
|
||||
}
|
||||
logger.info(`SFV "${sfvFile.filename}": ${entries.length} files listed, ${moving.length} moved into release ${releaseId} "${album}".`)
|
||||
return moving.length
|
||||
}
|
||||
|
||||
/** A new file an SFV already listed joins that SFV's release. Returns the release, or null. */
|
||||
/**
|
||||
* claim() for a batch of new files: one lookup for all their names, the
|
||||
* per-file work only for the few an SFV lists (seeding stores thousands
|
||||
* of files per batch; CONCEPT.md D37).
|
||||
*/
|
||||
async claimAll(files: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }[]): Promise<number> {
|
||||
if (files.length === 0) return 0
|
||||
const listed = new Set(
|
||||
(await db.from('sfv_entries').whereIn('name', [...new Set(files.map((f) => f.name))]).select('name')).map((e) => e.name as string)
|
||||
)
|
||||
let moved = 0
|
||||
for (const file of files) if (listed.has(file.name) && (await this.claim(file)) !== null) moved++
|
||||
return moved
|
||||
}
|
||||
|
||||
async claim(file: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }): Promise<number | null> {
|
||||
const entries = await db.from('sfv_entries').where('name', file.name)
|
||||
const entry = entries.find((e) => this.sameUpload(file, e.poster, e.posted_at ? new Date(e.posted_at) : null))
|
||||
if (!entry || Number(entry.release_id) === Number(file.release_id)) return null
|
||||
if (file.release_id) {
|
||||
const current = await db.from('releases').where('id', file.release_id).first()
|
||||
if (current && !WEAK_SOURCES.includes(current.source)) return null
|
||||
}
|
||||
await this.move([file.id], Number(entry.release_id))
|
||||
return Number(entry.release_id)
|
||||
}
|
||||
|
||||
private sameUpload(file: { poster?: string | null; date?: number | string | null }, poster: string | null, postedAt: Date | null) {
|
||||
if (poster && file.poster && file.poster !== poster) return false
|
||||
if (postedAt && file.date && Math.abs(Number(file.date) - postedAt.getTime()) > WINDOW_MS) return false
|
||||
return true
|
||||
}
|
||||
|
||||
private async move(fileIds: number[], releaseId: number) {
|
||||
if (fileIds.length === 0) return
|
||||
await db.from('files').whereIn('id', fileIds).update({ release_id: releaseId })
|
||||
await db.from('binaries').whereIn('file_id', fileIds).update({ release_id: releaseId })
|
||||
}
|
||||
}
|
||||
|
||||
export default new SfvService()
|
||||
@@ -1,45 +0,0 @@
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
|
||||
/** After a signal, give cleanup this long before exiting anyway. */
|
||||
const FORCE_EXIT_MS = 25_000
|
||||
|
||||
/**
|
||||
* Graceful shutdown for the long-running commands. SIGTERM (docker stop) and
|
||||
* SIGINT wake up `wait()` and `sleep()`, so workers can finish their active
|
||||
* jobs and close their connections before the process exits.
|
||||
*/
|
||||
class Shutdown {
|
||||
private readonly controller = new AbortController()
|
||||
|
||||
constructor() {
|
||||
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
||||
process.once(signal, () => {
|
||||
this.controller.abort()
|
||||
setTimeout(() => process.exit(0), FORCE_EXIT_MS).unref()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
get requested(): boolean {
|
||||
return this.controller.signal.aborted
|
||||
}
|
||||
|
||||
/** Resolves when a shutdown signal arrives. */
|
||||
wait(): Promise<void> {
|
||||
if (this.requested) return Promise.resolve()
|
||||
return new Promise((resolve) =>
|
||||
this.controller.signal.addEventListener('abort', () => resolve(), { once: true })
|
||||
)
|
||||
}
|
||||
|
||||
/** Sleeps, but returns early on shutdown. */
|
||||
async sleep(ms: number): Promise<void> {
|
||||
try {
|
||||
await sleep(ms, undefined, { signal: this.controller.signal })
|
||||
} catch {
|
||||
// aborted by a shutdown signal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new Shutdown()
|
||||
@@ -1,146 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import IngestService from '#services/IngestService'
|
||||
import NameService from '#services/NameService'
|
||||
import { splitVerified } from '#lib/split/index'
|
||||
|
||||
/** Conflicted binaries handled per run; the rest follow on the next tick. */
|
||||
const BATCH = 200
|
||||
|
||||
export interface SplitResult {
|
||||
parents: number
|
||||
volumes: number
|
||||
completed: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits conflicted obfuscated binaries (several equal-sized files under one
|
||||
* size+parts identity) into one binary per volume. Only volumes whose
|
||||
* combined part CRCs match the file CRC of their last part are created, so a
|
||||
* split can never produce a corrupt file; the rest waits for more articles.
|
||||
* See CONCEPT.md (D20).
|
||||
*/
|
||||
class SplitService {
|
||||
async splitPending(): Promise<SplitResult> {
|
||||
const parents = await db
|
||||
.from('binaries')
|
||||
.whereNotNull('conflicted_at')
|
||||
.whereNull('parent_binary_id')
|
||||
// split_at = 'infinity' marks binaries that cannot be split (see migration …810).
|
||||
.where((q) => q.whereNull('split_at').orWhereRaw('last_seen_at > split_at'))
|
||||
// Most recently active first: those are the posts still coming in.
|
||||
.orderBy('last_seen_at', 'desc')
|
||||
.limit(BATCH)
|
||||
|
||||
const volumeIds: number[] = []
|
||||
for (const parent of parents) {
|
||||
volumeIds.push(...(await this.split(parent)))
|
||||
}
|
||||
|
||||
// Former conflicts that turned out to be reposts (unflagged, keep their
|
||||
// split_at): no new article may arrive to trigger their completion.
|
||||
const unflagged = await db
|
||||
.from('binaries')
|
||||
.whereNull('conflicted_at')
|
||||
.whereNull('completed_at')
|
||||
.whereNull('parent_binary_id')
|
||||
.whereNotNull('split_at')
|
||||
.whereRaw("split_at <> 'infinity'")
|
||||
.limit(BATCH)
|
||||
.select('id')
|
||||
volumeIds.push(...unflagged.map((b) => Number(b.id)))
|
||||
|
||||
await IngestService.matchPar2Releases(volumeIds)
|
||||
await NameService.resolve(volumeIds)
|
||||
const completed = await IngestService.claimCompleted(volumeIds)
|
||||
await IngestService.enqueue(completed)
|
||||
return { parents: parents.length, volumes: volumeIds.length, completed: completed.length }
|
||||
}
|
||||
|
||||
/** Creates the newly verified volumes of one conflicted binary; returns their ids. */
|
||||
private async split(parent: any): Promise<number[]> {
|
||||
return db.transaction(async (trx) => {
|
||||
// Unverified volumes from before verification existed can never complete.
|
||||
await trx.from('binaries').where('parent_binary_id', parent.id).whereNull('completed_at').delete()
|
||||
|
||||
const { rows } = await trx.rawQuery(
|
||||
`WITH taken AS (
|
||||
SELECT p.message_id FROM parts p JOIN binaries v ON v.id = p.binary_id
|
||||
WHERE v.parent_binary_id = ?
|
||||
), candidates AS (
|
||||
SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM parts WHERE binary_id = ?
|
||||
UNION ALL
|
||||
SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM part_conflicts WHERE binary_id = ?
|
||||
)
|
||||
SELECT * FROM candidates WHERE message_id NOT IN (SELECT message_id FROM taken)`,
|
||||
[parent.id, parent.id, parent.id]
|
||||
)
|
||||
const volumes = splitVerified(
|
||||
rows.map((r: any) => ({
|
||||
number: r.number,
|
||||
messageId: r.message_id,
|
||||
bytes: r.bytes,
|
||||
articleNumber: r.article_number === null ? null : Number(r.article_number),
|
||||
pcrc32: r.pcrc32,
|
||||
partSize: r.part_size,
|
||||
fileCrc32: r.file_crc32,
|
||||
})),
|
||||
{ totalParts: parent.total_parts }
|
||||
)
|
||||
|
||||
const now = new Date()
|
||||
const ids: number[] = []
|
||||
for (const articles of volumes) {
|
||||
// A verified volume is fixed, so its first article identifies it.
|
||||
const fileKey = createHash('sha1').update(`${parent.file_key}#${articles[0].messageId}`).digest('hex')
|
||||
const inserted = await trx
|
||||
.knexClient('binaries')
|
||||
.insert({
|
||||
file_key: fileKey,
|
||||
poster: parent.poster,
|
||||
subject: parent.subject,
|
||||
name: parent.name,
|
||||
total_parts: parent.total_parts,
|
||||
groups: JSON.stringify(parent.groups),
|
||||
posted_at: parent.posted_at,
|
||||
size: parent.size,
|
||||
release_id: parent.release_id,
|
||||
parent_binary_id: parent.id,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
})
|
||||
.onConflict('file_key')
|
||||
.ignore()
|
||||
.returning('id')
|
||||
if (inserted.length === 0) continue
|
||||
const id = Number(inserted[0].id)
|
||||
await trx.knexClient('parts').insert(
|
||||
articles.map((a) => ({
|
||||
binary_id: id,
|
||||
number: a.number,
|
||||
message_id: a.messageId,
|
||||
bytes: a.bytes,
|
||||
article_number: a.articleNumber,
|
||||
pcrc32: a.pcrc32,
|
||||
part_size: a.partSize,
|
||||
file_crc32: a.fileCrc32,
|
||||
}))
|
||||
)
|
||||
// The fingerprint of a volume is that of its part 1 article.
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries SET md5_16k = (
|
||||
SELECT md5_16k FROM part_conflicts WHERE binary_id = ? AND message_id = ?
|
||||
UNION ALL SELECT md5_16k FROM parts WHERE binary_id = ? AND message_id = ? LIMIT 1)
|
||||
WHERE id = ?`,
|
||||
[parent.id, articles[0].messageId, parent.id, articles[0].messageId, id]
|
||||
)
|
||||
ids.push(id)
|
||||
}
|
||||
|
||||
await trx.from('binaries').where('id', parent.id).update({ split_at: now })
|
||||
return ids
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default new SplitService()
|
||||
@@ -1,493 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import NzbImportService from '#services/NzbImportService'
|
||||
import IngestService, { type OverviewHeader } from '#services/IngestService'
|
||||
import { parseNzb } from '#lib/nzb/index'
|
||||
import {
|
||||
moderationTarget,
|
||||
passwordFromSpot,
|
||||
spotterId,
|
||||
decodeSpotBinary,
|
||||
decodeSpotText,
|
||||
parseSpotArticleHeaders,
|
||||
parseSpotHeader,
|
||||
parseSpotXml,
|
||||
verifySpotHeader,
|
||||
type SpotHeader,
|
||||
type SpotXml,
|
||||
} from '#lib/spotnet/index'
|
||||
|
||||
/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
|
||||
const MAX_NZB_SEGMENTS = 20
|
||||
/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
|
||||
const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
|
||||
/** Rows per INSERT: 14 columns each, under Postgres' 65,535 bind parameters. */
|
||||
const INSERT_CHUNK = 2000
|
||||
/** Spot images larger than this many articles aren't fetched. */
|
||||
const MAX_IMAGE_SEGMENTS = 10
|
||||
/** A claimed spot whose worker died is taken again after this long. */
|
||||
const RECLAIM_AFTER = '1 hour'
|
||||
|
||||
/** The NZB's articles were read, but they don't decode to an NZB (e.g. "invalid code lengths set"). */
|
||||
export class NzbUnreadable extends Error {}
|
||||
|
||||
/**
|
||||
* A spot's title: from its XML, which is decoded properly (the Subject of a
|
||||
* raw header is read as ISO-8859-1), else from the header.
|
||||
*/
|
||||
function spotTitle(head: { header: SpotHeader; xml: SpotXml | null }): string {
|
||||
return head.xml?.title || head.header.title
|
||||
}
|
||||
|
||||
/**
|
||||
* A placeholder NZB (e.g. Dummy.rar + Dummy.nzb, 2.6 KB, shared by many
|
||||
* spots): less than 1% of the size the spot announces (the real NZB is
|
||||
* handed out elsewhere), or under 16 KB in all (text-only spots: requests,
|
||||
* announcements). Its files are not imported, and the spot is hidden.
|
||||
*/
|
||||
export function isDummyNzb(nzb: { files: { segments: { bytes: number }[] }[] }, spotSize: number): boolean {
|
||||
const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
|
||||
return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
|
||||
}
|
||||
|
||||
export function isSpotGroup(name: string): boolean {
|
||||
return indexerConfig.spotGroups.includes(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Spotnet (CONCEPT.md D29). Spots are signed posts in free.pt; their full
|
||||
* header holds XML with the message ids of a compressed NZB, which is
|
||||
* imported like a posted NZB (D26). Spot NZBs name many obfuscated posts
|
||||
* (nyuu, ngPost, the skipped multi-group uploader).
|
||||
*/
|
||||
class SpotService {
|
||||
/** Queues the spots among a batch of one spot group's headers. Returns how many. */
|
||||
async recordHeaders(groupName: string, overviews: [number, OverviewHeader][]): Promise<number> {
|
||||
const now = new Date()
|
||||
const rows = []
|
||||
for (const [articleNumber, header] of overviews) {
|
||||
if (!header?.from || !header['message-id']) continue
|
||||
const spot = parseSpotHeader(header.from, header.subject ?? '', header['message-id'], header.date)
|
||||
if (!spot?.title) continue
|
||||
rows.push({
|
||||
message_id: spot.messageId,
|
||||
group_name: groupName,
|
||||
article_number: Number(articleNumber),
|
||||
title: spot.title,
|
||||
tag: spot.tag || null,
|
||||
poster: spot.poster,
|
||||
category: spot.category,
|
||||
subcats: JSON.stringify(spot.subcats),
|
||||
size: spot.size || null,
|
||||
posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
|
||||
key_id: spot.keyId,
|
||||
spotter_id: spotterId(spot.selfSignedPubKey) || null,
|
||||
// From the overview; titles with accents may fail here (UTF-8 decoding)
|
||||
// and are verified again on their raw header by worker:spots.
|
||||
verified: verifySpotHeader(spot),
|
||||
created_at: now,
|
||||
})
|
||||
}
|
||||
for (let i = 0; i < rows.length; i += INSERT_CHUNK) {
|
||||
await db.knexQuery().table('spots').insert(rows.slice(i, i + INSERT_CHUNK)).onConflict('message_id').ignore()
|
||||
}
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes up to `limit` pending spots, marked `processing`: recent ones
|
||||
* (to be imported) first, then the history in posting order, so live spots
|
||||
* don't wait behind a backfill and moderation follows its targets.
|
||||
*/
|
||||
async claim(limit: number): Promise<any[]> {
|
||||
const claim = async (filter: string, bindings: unknown[], n: number) =>
|
||||
(
|
||||
await db.rawQuery(
|
||||
`UPDATE spots SET status = 'processing', processed_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots WHERE ${filter}
|
||||
ORDER BY article_number LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[...bindings, n]
|
||||
)
|
||||
).rows as any[]
|
||||
const rows = await claim(`status = 'pending' AND posted_at >= ?`, [this.importCutoff()], limit)
|
||||
if (rows.length < limit) rows.push(...(await claim(`status = 'pending'`, [], limit - rows.length)))
|
||||
if (rows.length < limit) {
|
||||
rows.push(...(await claim(`status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}'`, [], limit - rows.length)))
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
/** Spots posted before this are listed, not imported: their NZB is read on first download. */
|
||||
importCutoff(): Date {
|
||||
return new Date(Date.now() - indexerConfig.spotImportDays * 24 * 60 * 60 * 1000)
|
||||
}
|
||||
|
||||
/** Old, verified spots that aren't moderation: listed in bulk, without reading anything more. */
|
||||
isListable(spot: any): boolean {
|
||||
return (
|
||||
spot.verified === true &&
|
||||
spot.posted_at !== null &&
|
||||
new Date(spot.posted_at) < this.importCutoff() &&
|
||||
!moderationTarget({ keyId: spot.key_id, title: spot.title })
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the listing releases (source `spot`, no files) for these spots.
|
||||
* The release key is the one an import of the spot's NZB uses, so the
|
||||
* import on first download fills this release.
|
||||
*/
|
||||
async list(spots: any[]): Promise<number> {
|
||||
if (spots.length === 0) return 0
|
||||
const keyOf = (spot: any) => createHash('sha1').update(`nzb\n${spot.message_id}`).digest('hex')
|
||||
const ids = await IngestService.upsertReleases(
|
||||
db.connection().getWriteClient(),
|
||||
spots.map((spot) => ({
|
||||
release: { key: keyOf(spot), name: spot.title, source: 'spot' as const },
|
||||
poster: spot.poster,
|
||||
groups: [spot.group_name],
|
||||
postedAt: spot.posted_at ? new Date(spot.posted_at) : undefined,
|
||||
}))
|
||||
)
|
||||
const rows = spots.map((spot) => ({ message_id: spot.message_id, release_id: ids.get(keyOf(spot)), size: spot.size }))
|
||||
await db.rawQuery(
|
||||
`UPDATE releases r SET size = v.size FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, size bigint)
|
||||
WHERE r.id = v.release_id AND r.file_count = 0 AND v.size IS NOT NULL`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE spots s SET status = 'listed', release_id = v.release_id, processed_at = now()
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, release_id bigint)
|
||||
WHERE s.message_id = v.message_id`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies the spot on its raw header (the signature covers ISO-8859-1
|
||||
* bytes, which the overview, read as UTF-8, may have mangled), then
|
||||
* applies moderation, lists an old spot or imports a recent one's NZB.
|
||||
* Returns the new status.
|
||||
*/
|
||||
async process(spot: any): Promise<string> {
|
||||
const update: Record<string, unknown> = { processed_at: new Date(), error: null }
|
||||
try {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) {
|
||||
update.status = 'unverified'
|
||||
} else if (moderationTarget(head.header)) {
|
||||
update.status = 'moderation'
|
||||
await this.moderate(head.header, moderationTarget(head.header)!)
|
||||
} else {
|
||||
update.title = spotTitle(head)
|
||||
update.poster = head.header.poster
|
||||
update.spotter_id = spotterId(head.header.selfSignedPubKey) || null
|
||||
Object.assign(update, this.details(head.xml))
|
||||
if (new Date(head.header.stamp * 1000) < this.importCutoff()) {
|
||||
await this.list([{ ...spot, title: spotTitle(head), poster: head.header.poster }])
|
||||
update.status = 'listed'
|
||||
} else {
|
||||
Object.assign(update, await this.importNzb(spot.message_id, head))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
update.status = 'failed'
|
||||
update.error = String(error.message).slice(0, 500)
|
||||
logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
if (update.status === 'imported' || update.status === 'listed') await this.applyPassword(spot.message_id)
|
||||
return update.status as string
|
||||
}
|
||||
|
||||
/**
|
||||
* The NZB of a listing, on first download: reads and imports it. Returns
|
||||
* the release that now holds its files (the listing's own, unless the NZB
|
||||
* joined files already known elsewhere), or null.
|
||||
*/
|
||||
async importListing(releaseId: number): Promise<number | null> {
|
||||
const spot = await db.from('spots').where('release_id', releaseId).where('status', 'listed').first()
|
||||
if (!spot) return null
|
||||
return this.reimport(spot)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a spot's NZB again and imports it, so its files end up in the
|
||||
* spot's own release under the spot's title. Returns that release, or null.
|
||||
*/
|
||||
async reimport(spot: any): Promise<number | null> {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) return null
|
||||
const result = await this.importNzb(spot.message_id, head)
|
||||
await db
|
||||
.from('spots')
|
||||
.where('message_id', spot.message_id)
|
||||
.update({ ...result, ...this.details(head.xml), processed_at: new Date() })
|
||||
await this.applyPassword(spot.message_id)
|
||||
return (result.release_id as number | undefined) ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* A password the spotter wrote in the spot's title or description (not in
|
||||
* the NZB) goes on its release, unless the release has one already.
|
||||
*/
|
||||
async applyPassword(messageId: string): Promise<string | null> {
|
||||
const spot = await db.from('spots').where('message_id', messageId).whereNotNull('release_id').first()
|
||||
const password = spot ? passwordFromSpot(spot.title, spot.description) : null
|
||||
if (!password) return null
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', spot.release_id)
|
||||
.whereNull('password')
|
||||
.update({ password, password_source: 'spot' })
|
||||
return password
|
||||
}
|
||||
|
||||
/** Columns for what the spot's XML adds to its overview: description, image, website. */
|
||||
private details(xml: SpotXml | null): Record<string, unknown> {
|
||||
return {
|
||||
description: xml?.description || null,
|
||||
image: xml?.image ? JSON.stringify(xml.image) : null,
|
||||
...(xml?.website ? { website: xml.website } : {}),
|
||||
details_at: new Date(),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A spot row with its description and image, reading its full header
|
||||
* first when that hasn't happened yet (old spots are listed from their
|
||||
* overview only). Returns the row as it is when Usenet can't be reached.
|
||||
*/
|
||||
async withDetails(spot: any): Promise<any> {
|
||||
if (spot.details_at) return spot
|
||||
try {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
const update = head ? this.details(head.xml) : { details_at: new Date() }
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
await this.applyPassword(spot.message_id)
|
||||
return { ...spot, ...update, image: head?.xml?.image ?? null }
|
||||
} catch (error: any) {
|
||||
logger.warn(`Spot ${spot.message_id} details: ${error.message}`)
|
||||
// The article is gone (430): nothing to read later either.
|
||||
if (/^4\d\d/.test(String(error.message))) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
}
|
||||
return spot
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the details of up to `limit` imported spots that don't have them
|
||||
* (imported before details were kept), newest first. Returns how many.
|
||||
* Listed spots are left to their first view: there are millions.
|
||||
* Returns how many were read (0 when Usenet can't be reached).
|
||||
*/
|
||||
async fillDetails(limit: number): Promise<number> {
|
||||
const spots = await db
|
||||
.from('spots')
|
||||
.where('status', 'imported')
|
||||
.whereNull('details_at')
|
||||
.orderBy('posted_at', 'desc')
|
||||
.limit(limit)
|
||||
let read = 0
|
||||
for (const spot of spots) if ((await this.withDetails(spot)).details_at) read++
|
||||
return read
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes up to `limit` listed spots whose article and NZB haven't been read,
|
||||
* newest first, marking them so another reader skips them.
|
||||
*/
|
||||
async claimListings(limit: number): Promise<any[]> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE spots SET nzb_read_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots WHERE status = 'listed' AND nzb_read_at IS NULL
|
||||
ORDER BY posted_at DESC LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[limit]
|
||||
)
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a listed spot's article and NZB without importing its files
|
||||
* (LOG.md 2026-10-08 (10)): the details (description, image), the NZB's
|
||||
* own password (`password_source = 'nzb'`, over one from the spot's text),
|
||||
* else one in the spot's text; a placeholder NZB hides the spot (`dummy`).
|
||||
* Returns what it found. A connection failure gives the spot back for later.
|
||||
*/
|
||||
async readListing(spot: any): Promise<string> {
|
||||
try {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
return 'unverified'
|
||||
}
|
||||
const update: Record<string, unknown> = this.details(head.xml)
|
||||
let outcome = 'nothing'
|
||||
let nzbXml: string | null = null
|
||||
try {
|
||||
nzbXml = await this.readNzb(head)
|
||||
} catch (error: any) {
|
||||
if (error instanceof NzbUnreadable) {
|
||||
// Corrupt: reading it again won't help (LOG.md 2026-10-08 (12)).
|
||||
update.error = `NZB unreadable: ${error.message}`.slice(0, 500)
|
||||
outcome = 'nzb-unreadable'
|
||||
} else if (/^4\d\d/.test(String(error.message))) {
|
||||
// The NZB's articles are gone: the listing can't be downloaded any more.
|
||||
update.error = 'NZB articles gone'
|
||||
outcome = 'nzb-gone'
|
||||
} else {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
const nzb = nzbXml ? parseNzb(nzbXml) : null
|
||||
if (nzb && nzb.files.length > 0 && isDummyNzb(nzb, head.header.size)) {
|
||||
update.status = 'dummy'
|
||||
outcome = 'dummy'
|
||||
} else if (nzb?.meta.password && spot.release_id) {
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', spot.release_id)
|
||||
.where((q) => q.whereNull('password').orWhere('password_source', 'spot'))
|
||||
.update({ password: nzb.meta.password, password_source: 'nzb' })
|
||||
outcome = 'nzb-password'
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
if (outcome !== 'dummy' && (await this.applyPassword(spot.message_id)) && outcome !== 'nzb-password') outcome = 'spot-password'
|
||||
return outcome
|
||||
} catch (error: any) {
|
||||
// The spot's own article is gone (430): nothing to read later either.
|
||||
if (/^4\d\d/.test(String(error.message))) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
return 'gone'
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ nzb_read_at: null })
|
||||
logger.warn(`Spot ${spot.message_id} listing: ${error.message}`)
|
||||
return 'retry'
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads up to `limit` unread listings, `concurrency` at a time. Returns the outcomes counted. */
|
||||
async readListings(limit: number, concurrency: number): Promise<Record<string, number>> {
|
||||
const spots = await this.claimListings(limit)
|
||||
const counts: Record<string, number> = {}
|
||||
let next = 0
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.max(1, concurrency) }, async () => {
|
||||
while (next < spots.length) {
|
||||
const outcome = await this.readListing(spots[next++])
|
||||
counts[outcome] = (counts[outcome] ?? 0) + 1
|
||||
}
|
||||
})
|
||||
)
|
||||
return counts
|
||||
}
|
||||
|
||||
/** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
|
||||
async image(spot: any): Promise<Buffer | null> {
|
||||
const segments: string[] = spot.image?.segments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_IMAGE_SEGMENTS) return null
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
return decodeSpotBinary(bodies, false)
|
||||
}
|
||||
|
||||
/** The spot's raw header, verified; null when it doesn't verify. */
|
||||
private async readHead(messageId: string): Promise<{ header: SpotHeader; xml: SpotXml | null } | null> {
|
||||
const raw = await this.fetch(`HEAD <${messageId}>`)
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
|
||||
const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
|
||||
if (!header || !verifySpotHeader(header)) return null
|
||||
return { header, xml: parseSpotXml(decodeSpotText(xml)) }
|
||||
}
|
||||
|
||||
/** Downloads and decodes the spot's compressed NZB; null when it has none (or a too large one). */
|
||||
private async readNzb(head: { xml: SpotXml | null }): Promise<string | null> {
|
||||
const segments = head.xml?.nzbSegments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return null
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
try {
|
||||
return decodeSpotBinary(bodies).toString('utf8')
|
||||
} catch (error: any) {
|
||||
throw new NzbUnreadable(String(error.message))
|
||||
}
|
||||
}
|
||||
|
||||
/** Downloads the spot's compressed NZB and imports it as a release (D26). */
|
||||
private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
|
||||
const nzbXml = await this.readNzb(head)
|
||||
if (!nzbXml) return { status: 'no-nzb' }
|
||||
const nzb = parseNzb(nzbXml)
|
||||
if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
|
||||
const { header } = head
|
||||
if (isDummyNzb(nzb, header.size)) return { status: 'dummy' }
|
||||
const title = spotTitle(head)
|
||||
const result = await NzbImportService.import(
|
||||
{
|
||||
id: null,
|
||||
filename: `${title}.nzb`,
|
||||
poster: header.poster,
|
||||
groups: [...new Set(nzb.files.flatMap((f) => f.groups))],
|
||||
date: header.stamp * 1000,
|
||||
first_message_id: messageId,
|
||||
},
|
||||
nzbXml,
|
||||
{ name: title, source: 'spot', moveContainer: false, trusted: true }
|
||||
)
|
||||
return result ? { status: 'imported', release_id: result.releaseId } : { status: 'no-nzb' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a delete/dispose/remove: the target spot is marked `removed`
|
||||
* (its release is then hidden from search). As in Spotweb, a personal
|
||||
* dispose must come from the same spotter within 5 days; a moderator's
|
||||
* (signed with Spotnet's key, no personal key) always counts.
|
||||
*/
|
||||
private async moderate(header: SpotHeader, targetId: string) {
|
||||
const target = await db.from('spots').where('message_id', targetId).first()
|
||||
if (!target) return
|
||||
const personal = header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : ''
|
||||
if (personal) {
|
||||
const age = header.stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
|
||||
if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
|
||||
}
|
||||
await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
|
||||
logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
|
||||
// Spots that joined its release (a correction, a repost) take the files over now.
|
||||
if (target.release_id) {
|
||||
const sharing = await db
|
||||
.from('spots')
|
||||
.where('release_id', target.release_id)
|
||||
.where('status', 'imported')
|
||||
.whereNot('message_id', targetId)
|
||||
for (const spot of sharing) await this.reimport(spot)
|
||||
}
|
||||
}
|
||||
|
||||
/** One HEAD or BODY, as raw bytes. */
|
||||
private async fetch(command: string): Promise<Buffer> {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
const { data } = await NntpService.timed<any>(conn._artcmd(command), command)
|
||||
return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
|
||||
} catch (error: any) {
|
||||
// 430 (no such article) is an answer, not a broken connection.
|
||||
failed = !/^4\d\d/.test(String(error.message))
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new SpotService()
|
||||
@@ -1,252 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import IngestService, { type IngestSegment } from '#services/IngestService'
|
||||
import { counterFileKey } from '#lib/subject/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
|
||||
/** Rows per statement (a key and a JSON array each). */
|
||||
const CHUNK = 5000
|
||||
|
||||
/**
|
||||
* Staging of lone header segments (CONCEPT.md D35). Uploads that give every
|
||||
* article its own random subject turned each article into a binary, a part
|
||||
* and a release, which GC deleted two hours later: ~3.9M binaries an hour
|
||||
* against ~123k real files. A segment whose file is new and alone in its
|
||||
* fetch batch now waits in `staged_segments` (unlogged, one index) until a
|
||||
* second segment of that file arrives; only then do both go into the real
|
||||
* tables. GC drops what waited longer than STAGING_TTL_SECONDS (30 min).
|
||||
*
|
||||
* Counter grouping (D36): those random-subject uploads still keep poster,
|
||||
* file counter `[i/M]` and part count per file. Staged segments also carry
|
||||
* that key (`counter_key`); once two staged segments from different subjects
|
||||
* share it, they become one file under that key, and later segments with it
|
||||
* join that file directly.
|
||||
*/
|
||||
class StagingService {
|
||||
enabled(groupName: string): boolean {
|
||||
const groups = indexerConfig.stagingGroups
|
||||
return groups.includes('*') || groups.includes(groupName)
|
||||
}
|
||||
|
||||
/** Whether counter grouping (D36) applies to this group. */
|
||||
countersEnabled(groupName: string): boolean {
|
||||
const groups = indexerConfig.counterGroupingGroups
|
||||
return groups.includes('*') || groups.includes(groupName)
|
||||
}
|
||||
|
||||
/**
|
||||
* The segments of a header batch that should be ingested now: the batch's
|
||||
* own, minus lone segments of new files (staged instead), plus staged
|
||||
* segments whose file this batch shows to be real. `staged` counts the
|
||||
* segments that went to staging.
|
||||
*/
|
||||
async filter(segments: IngestSegment[], { counters = false } = {}): Promise<{ ingest: IngestSegment[]; staged: number }> {
|
||||
const byKey = new Map<string, IngestSegment[]>()
|
||||
for (const segment of segments) {
|
||||
const list = byKey.get(segment.fileKey)
|
||||
if (list) list.push(segment)
|
||||
else byKey.set(segment.fileKey, [segment])
|
||||
}
|
||||
|
||||
const ingest: IngestSegment[] = []
|
||||
const lone: IngestSegment[] = []
|
||||
for (const list of byKey.values()) {
|
||||
// Single-segment files are complete; two different articles of a file in one batch make it real.
|
||||
// (One article twice is a crosspost, LOG.md 2026-10-02.)
|
||||
const articles = new Set(list.map((s) => s.messageId)).size
|
||||
if (list[0].totalParts <= 1 || articles >= 2 || list[0].weakIdentity || list[0].size != null) ingest.push(...list)
|
||||
else lone.push(list[0])
|
||||
}
|
||||
|
||||
const counterOf = new Map<IngestSegment, string>()
|
||||
if (counters) {
|
||||
for (const segment of lone) {
|
||||
const key = counterFileKey(segment)
|
||||
if (key) counterOf.set(segment, key)
|
||||
}
|
||||
}
|
||||
|
||||
// Files we already have a binary for (by subject, or by counters) take their segments directly.
|
||||
const known = new Set<string>()
|
||||
const lookup = [...new Set([...lone.map((s) => s.fileKey), ...counterOf.values()])]
|
||||
for (let i = 0; i < lookup.length; i += CHUNK) {
|
||||
const rows = await db.from('binaries').whereIn('file_key', lookup.slice(i, i + CHUNK)).select('file_key')
|
||||
for (const row of rows) known.add(row.file_key)
|
||||
}
|
||||
const toStage: IngestSegment[] = []
|
||||
const knownCounters = new Set<string>()
|
||||
for (const segment of lone) {
|
||||
const counter = counterOf.get(segment)
|
||||
if (known.has(segment.fileKey)) {
|
||||
ingest.push(segment)
|
||||
} else if (counter && known.has(counter)) {
|
||||
ingest.push(asCounterFile(segment, counter))
|
||||
knownCounters.add(counter)
|
||||
} else {
|
||||
toStage.push(segment)
|
||||
}
|
||||
}
|
||||
|
||||
// Stage the rest; a file that now has two staged segments comes out again.
|
||||
const promoted: string[] = []
|
||||
// Key order, so concurrent jobs lock rows in the same order.
|
||||
toStage.sort((a, b) => a.fileKey.localeCompare(b.fileKey))
|
||||
for (let i = 0; i < toStage.length; i += CHUNK) {
|
||||
const rows = toStage.slice(i, i + CHUNK).map((s) => ({
|
||||
file_key: s.fileKey,
|
||||
counter_key: counterOf.get(s) ?? null,
|
||||
segments: [serialize(s)],
|
||||
staged_at: new Date(),
|
||||
}))
|
||||
const { rows: result } = await retryDeadlock(() => db.rawQuery(
|
||||
`INSERT INTO staged_segments (file_key, counter_key, segments, staged_at)
|
||||
SELECT file_key, counter_key, segments, staged_at
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(file_key text, counter_key text, segments jsonb, staged_at timestamptz)
|
||||
ON CONFLICT (file_key) DO UPDATE SET segments = CASE
|
||||
-- The same article again (a crosspost, or a re-fetched range) is no second segment.
|
||||
WHEN EXISTS (SELECT 1 FROM jsonb_array_elements(staged_segments.segments) e
|
||||
WHERE e->>'messageId' = EXCLUDED.segments->0->>'messageId') THEN staged_segments.segments
|
||||
ELSE staged_segments.segments || EXCLUDED.segments END
|
||||
RETURNING file_key, jsonb_array_length(segments) AS n`,
|
||||
[JSON.stringify(rows)]
|
||||
))
|
||||
for (const row of result) if (Number(row.n) >= 2) promoted.push(row.file_key)
|
||||
}
|
||||
|
||||
// Counter files: two staged rows (two subjects) with one counter key make a file.
|
||||
const stagedCounters = [...new Set(toStage.map((s) => counterOf.get(s)).filter((k): k is string => Boolean(k)))]
|
||||
const readyCounters: string[] = []
|
||||
for (let i = 0; i < stagedCounters.length; i += CHUNK) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT counter_key FROM staged_segments WHERE counter_key = ANY(?) GROUP BY counter_key HAVING count(*) >= 2`,
|
||||
[stagedCounters.slice(i, i + CHUNK)]
|
||||
)
|
||||
for (const row of rows) readyCounters.push(row.counter_key)
|
||||
}
|
||||
|
||||
// Promoted files, staged segments of files this batch ingests anyway, and ready counter files.
|
||||
const fileKeys = [...new Set([...promoted, ...ingest.map((s) => s.fileKey)])]
|
||||
const counterKeys = [...new Set([...readyCounters, ...knownCounters])]
|
||||
const seen = new Set(ingest.map((s) => s.messageId))
|
||||
for (const segment of await this.take(fileKeys, counterKeys)) {
|
||||
if (!seen.has(segment.messageId)) ingest.push(segment)
|
||||
seen.add(segment.messageId)
|
||||
}
|
||||
const staged = toStage.filter((s) => !seen.has(s.messageId)).length
|
||||
return { ingest, staged }
|
||||
}
|
||||
|
||||
/**
|
||||
* GC: drops staged segments that waited longer than `maxAgeSeconds`, then
|
||||
* ingests those whose file got a binary meanwhile (two fetch jobs racing).
|
||||
* Expiry comes first so the table stays bounded whatever the budget; the
|
||||
* adoption starts from recently seen binaries (index on last_seen_at,
|
||||
* which moves in 10-minute steps) rather than joining every staged row.
|
||||
*/
|
||||
async sweep(maxAgeSeconds: number, budget: number): Promise<{ adopted: number; expired: number }> {
|
||||
let adopted = 0
|
||||
let expired = 0
|
||||
while (Date.now() < budget) {
|
||||
const result = await db.rawQuery(
|
||||
`DELETE FROM staged_segments WHERE ctid IN (
|
||||
SELECT ctid FROM staged_segments WHERE staged_at < now() - make_interval(secs => ?) LIMIT 20000)`,
|
||||
[maxAgeSeconds]
|
||||
)
|
||||
expired += result.rowCount
|
||||
if (result.rowCount < 20000) break
|
||||
}
|
||||
let lastId = 0
|
||||
while (Date.now() < budget) {
|
||||
const recent = await db
|
||||
.from('binaries')
|
||||
.where('last_seen_at', '>', new Date(Date.now() - (maxAgeSeconds + 15 * 60) * 1000))
|
||||
.where('id', '>', lastId)
|
||||
.orderBy('id')
|
||||
.limit(CHUNK)
|
||||
.select('id', 'file_key')
|
||||
if (recent.length === 0) break
|
||||
lastId = Number(recent[recent.length - 1].id)
|
||||
const keys = recent.map((b) => b.file_key as string)
|
||||
// A binary's key is a subject key or a counter key; staged rows are looked up by both.
|
||||
const segments = await this.take(keys, keys)
|
||||
if (segments.length > 0) await IngestService.ingest(segments)
|
||||
adopted += segments.length
|
||||
}
|
||||
return { adopted, expired }
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes and returns the staged segments of these files: by subject key,
|
||||
* or by counter key (returned as segments of the counter file).
|
||||
*/
|
||||
private async take(fileKeys: string[], counterKeys: string[] = []): Promise<IngestSegment[]> {
|
||||
const taken: IngestSegment[] = []
|
||||
for (let i = 0; i < fileKeys.length; i += CHUNK) {
|
||||
const { rows } = await retryDeadlock(() =>
|
||||
db.rawQuery(`DELETE FROM staged_segments WHERE file_key = ANY(?) RETURNING segments`, [fileKeys.slice(i, i + CHUNK)])
|
||||
)
|
||||
for (const row of rows) taken.push(...(row.segments as any[]).map(deserialize))
|
||||
}
|
||||
for (let i = 0; i < counterKeys.length; i += CHUNK) {
|
||||
const { rows } = await retryDeadlock(() =>
|
||||
db.rawQuery(`DELETE FROM staged_segments WHERE counter_key = ANY(?) RETURNING counter_key, segments`, [counterKeys.slice(i, i + CHUNK)])
|
||||
)
|
||||
for (const row of rows) taken.push(...(row.segments as any[]).map((s) => asCounterFile(deserialize(s), row.counter_key)))
|
||||
}
|
||||
return taken
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A segment as part of its counter file (D36): the counter key is its file
|
||||
* key, it is a weak identity (a repeated part number from another upload
|
||||
* marks the binary conflicted), and its release is a weak one of its own,
|
||||
* which the upload's par2 set can gather later.
|
||||
*/
|
||||
function asCounterFile(segment: IngestSegment, counterKey: string): IngestSegment {
|
||||
return {
|
||||
...segment,
|
||||
fileKey: counterKey,
|
||||
weakIdentity: true,
|
||||
release: {
|
||||
key: createHash('sha1').update(`counter-release\n${counterKey}`).digest('hex'),
|
||||
name: fileStem(segment.name),
|
||||
source: 'weak-stem',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a staging statement again when it lost a deadlock: concurrent fetch
|
||||
* jobs insert by file key and delete by counter key, so their row locks can
|
||||
* cross. Postgres rolled the statement back, and each one is safe to repeat.
|
||||
*/
|
||||
async function retryDeadlock<T>(statement: () => Promise<T>, attempts = 5): Promise<T> {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
return await statement()
|
||||
} catch (error: any) {
|
||||
if (error?.code !== '40P01' || attempt >= attempts) throw error
|
||||
await new Promise((resolve) => setTimeout(resolve, 50 * attempt + Math.random() * 100))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function serialize(segment: IngestSegment) {
|
||||
return {
|
||||
...segment,
|
||||
postedAt: segment.postedAt?.toISOString(),
|
||||
articleNumber: segment.articleNumber?.toString(),
|
||||
}
|
||||
}
|
||||
|
||||
function deserialize(stored: any): IngestSegment {
|
||||
return {
|
||||
...stored,
|
||||
postedAt: stored.postedAt ? new Date(stored.postedAt) : undefined,
|
||||
articleNumber: stored.articleNumber ?? undefined,
|
||||
}
|
||||
}
|
||||
|
||||
export default new StagingService()
|
||||
+1
-7
@@ -12,18 +12,12 @@
|
||||
*/
|
||||
|
||||
await import('reflect-metadata')
|
||||
const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
|
||||
const { Ignitor, prettyPrintError } = await import('@adonisjs/core/ignitor')
|
||||
|
||||
/**
|
||||
* URL to the application root. AdonisJS need it to resolve
|
||||
* paths to file and directories for scaffolding commands
|
||||
*/
|
||||
/**
|
||||
* The command being run, for start/env.ts: Ace takes the arguments out of
|
||||
* process.argv before the environment is loaded.
|
||||
*/
|
||||
process.env.ACE_COMMAND ??= process.argv[2] ?? ''
|
||||
|
||||
const APP_ROOT = new URL('../', import.meta.url)
|
||||
|
||||
/**
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
*/
|
||||
|
||||
await import('reflect-metadata')
|
||||
const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
|
||||
const { Ignitor, prettyPrintError } = await import('@adonisjs/core/ignitor')
|
||||
|
||||
/**
|
||||
* URL to the application root. AdonisJS need it to resolve
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@
|
||||
process.env.NODE_ENV = 'test'
|
||||
|
||||
import 'reflect-metadata'
|
||||
import { Ignitor, prettyPrintError } from '@adonisjs/core'
|
||||
import { Ignitor, prettyPrintError } from '@adonisjs/core/ignitor'
|
||||
import { configure, processCLIArgs, run } from '@japa/runner'
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NntpService from '#services/NntpService'
|
||||
import ArchiveWalkService from '#services/ArchiveWalkService'
|
||||
|
||||
const BATCH = 5000
|
||||
|
||||
/**
|
||||
* Complete archive listings for releases inspected before archive walks
|
||||
* existed (CONCEPT.md D40): queues those whose listing leaves most of the
|
||||
* release unaccounted for. worker:collection walks them, one at a time and
|
||||
* behind inspection. With release ids, walks those right here instead.
|
||||
*/
|
||||
export default class ArchiveWalk extends BaseCommand {
|
||||
public static commandName = 'content:walk-archives'
|
||||
public static description = 'Queues releases with partial archive listings for a (low priority) header walk.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@args.spread({ description: 'Release ids to walk now (instead of queueing)', required: false })
|
||||
declare releaseIds?: string[]
|
||||
|
||||
@flags.number({ description: 'Queue at most this many releases' })
|
||||
declare limit?: number
|
||||
|
||||
@flags.boolean({ description: 'Also releases that were walked before' })
|
||||
declare again: boolean
|
||||
|
||||
public async run() {
|
||||
try {
|
||||
if (this.releaseIds?.length) {
|
||||
for (const id of this.releaseIds) {
|
||||
this.logger.info(JSON.stringify(await ArchiveWalkService.walk(Number(id))))
|
||||
}
|
||||
return
|
||||
}
|
||||
let lastId = 0
|
||||
let queued = 0
|
||||
while (!this.limit || queued < this.limit) {
|
||||
const query = db
|
||||
.from('releases as r')
|
||||
.where('r.id', '>', lastId)
|
||||
.whereNotNull('r.contents')
|
||||
.orderBy('r.id')
|
||||
.limit(BATCH)
|
||||
.select('r.id', 'r.size', 'r.contents_walked_at', db.raw(`(
|
||||
SELECT count(*) FROM jsonb_array_elements(r.contents)) AS n`), db.raw(`(
|
||||
SELECT coalesce(sum((e->>'size')::bigint), 0) FROM jsonb_array_elements(r.contents) e) AS listed`))
|
||||
const rows = await query
|
||||
if (rows.length === 0) break
|
||||
lastId = Number(rows.at(-1).id)
|
||||
const jobs = rows
|
||||
.filter((r) => this.again || !r.contents_walked_at)
|
||||
.filter((r) => ArchiveWalkService.worthWalking(Number(r.n), Number(r.listed), Number(r.size)))
|
||||
.slice(0, this.limit ? this.limit - queued : undefined)
|
||||
.map((r) => ({ name: 'walk', data: { releaseId: Number(r.id) }, opts: { jobId: `walk-${r.id}-${Date.now()}` } }))
|
||||
if (jobs.length > 0) await QueueService.archiveWalkQueue.addBulk(jobs)
|
||||
queued += jobs.length
|
||||
}
|
||||
this.logger.info(`Queued ${queued} releases for an archive walk.`)
|
||||
} finally {
|
||||
await QueueService.closeAll()
|
||||
await NntpService.shutdown()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import QueueService from '#services/QueueService'
|
||||
import RedisService from '#services/RedisService'
|
||||
import DeferredService, { isDeferred } from '#services/DeferredService'
|
||||
|
||||
const BATCH = 2000
|
||||
|
||||
/** Rebuilds the wait list (KEYS[1]) without the ids in the set KEYS[2], in one atomic pass. */
|
||||
const FILTER_LIST = `
|
||||
local ids = redis.call('LRANGE', KEYS[1], 0, -1)
|
||||
local keep = {}
|
||||
for _, id in ipairs(ids) do
|
||||
if redis.call('SISMEMBER', KEYS[2], id) == 0 then keep[#keep + 1] = id end
|
||||
end
|
||||
redis.call('DEL', KEYS[1])
|
||||
for i = 1, #keep, 5000 do
|
||||
redis.call('RPUSH', KEYS[1], unpack(keep, i, math.min(i + 4999, #keep)))
|
||||
end
|
||||
return #ids - #keep`
|
||||
|
||||
/**
|
||||
* Removes waiting body jobs whose message-id is on the skip or defer list,
|
||||
* e.g. after adding a pattern (CONCEPT.md D24, D28). Deferred ones are
|
||||
* recorded in deferred_files first, so they can still be processed later.
|
||||
*/
|
||||
export default class BodyQueuePrune extends BaseCommand {
|
||||
public static commandName = 'body-queue:prune'
|
||||
public static description = 'Drops waiting body jobs that are now skipped or deferred.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only count' })
|
||||
declare dryRun: boolean
|
||||
|
||||
@flags.boolean({ description: 'Drop deferred jobs without recording them (already recorded by an earlier run)' })
|
||||
declare skipRecord: boolean
|
||||
|
||||
public async run() {
|
||||
const redis = RedisService.client
|
||||
const queue = QueueService.bodyQueue
|
||||
const waitKey = queue.toKey('wait')
|
||||
let skipped = 0
|
||||
let deferred = 0
|
||||
let scanned = 0
|
||||
const dropped: string[] = []
|
||||
try {
|
||||
const ids: string[] = await redis.lrange(waitKey, 0, -1)
|
||||
for (let i = 0; i < ids.length; i += BATCH) {
|
||||
const batch = ids.slice(i, i + BATCH)
|
||||
const data: [Error | null, string | null][] = await batch
|
||||
.reduce((pipeline, id) => pipeline.hget(queue.toKey(id), 'data'), redis.pipeline())
|
||||
.exec()
|
||||
const drop: string[] = []
|
||||
const toDefer = new Map<string, { articleNumber: number; header: Record<string, string> }[]>()
|
||||
batch.forEach((id, j) => {
|
||||
const job = data[j][1] ? JSON.parse(data[j][1]!) : null
|
||||
if (!job?.header) return
|
||||
const messageId = job.header['message-id'] ?? ''
|
||||
if (isDeferred(job.header)) {
|
||||
const list = toDefer.get(job.group) ?? []
|
||||
list.push({ articleNumber: Number(job.articleNumber), header: job.header })
|
||||
toDefer.set(job.group, list)
|
||||
drop.push(id)
|
||||
deferred++
|
||||
} else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(messageId))) {
|
||||
drop.push(id)
|
||||
skipped++
|
||||
}
|
||||
})
|
||||
scanned += batch.length
|
||||
if (this.dryRun || drop.length === 0) continue
|
||||
if (!this.skipRecord) for (const [group, articles] of toDefer) await DeferredService.record(group, articles)
|
||||
dropped.push(...drop)
|
||||
}
|
||||
if (dropped.length > 0) {
|
||||
const dropKey = queue.toKey('prune-ids')
|
||||
await redis.del(dropKey)
|
||||
for (let i = 0; i < dropped.length; i += BATCH) await redis.sadd(dropKey, ...dropped.slice(i, i + BATCH))
|
||||
await redis.eval(FILTER_LIST, 2, waitKey, dropKey)
|
||||
await redis.del(dropKey)
|
||||
for (let i = 0; i < dropped.length; i += BATCH) {
|
||||
await redis.unlink(...dropped.slice(i, i + BATCH).map((id) => queue.toKey(id)))
|
||||
}
|
||||
}
|
||||
this.logger.info(
|
||||
`Scanned ${scanned} waiting body jobs: ${this.dryRun ? 'would drop' : 'dropped'} ${deferred} deferred and ${skipped} skipped.`
|
||||
)
|
||||
} finally {
|
||||
await QueueService.closeAll()
|
||||
await RedisService.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,91 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import QueueService from '#services/QueueService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import { isArchiveStart } from '#lib/content/index'
|
||||
import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
|
||||
|
||||
const BATCH = 1000
|
||||
|
||||
/**
|
||||
* Queues completed files for content inspection (names and groupings from
|
||||
* par2 and RAR headers, CONCEPT.md D21) that were completed before it
|
||||
* existed. Costs one article download per file, done by worker:collection.
|
||||
*/
|
||||
export default class ContentBackfill extends BaseCommand {
|
||||
public static commandName = 'content:backfill'
|
||||
public static description = 'Queues existing files with random names for content inspection.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.number({ description: 'Queue at most this many files' })
|
||||
declare limit?: number
|
||||
|
||||
@flags.boolean({ description: 'Also files that were inspected before (after release:rebuild --reset-content)' })
|
||||
declare all: boolean
|
||||
|
||||
@flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
|
||||
declare archiveNzbs: boolean
|
||||
|
||||
@flags.boolean({ description: 'Only unlisted 7z archives (first volumes: .7z, .7z.001)' })
|
||||
declare sevenZip: boolean
|
||||
|
||||
@flags.boolean({ description: 'Archive starts never inspected (RAR first volumes, 7z, .bin), also files from NZBs/spots' })
|
||||
declare archives: boolean
|
||||
|
||||
@flags.boolean({ description: 'SFV files not read yet (their listings group the files they name, D34)' })
|
||||
declare sfv: boolean
|
||||
|
||||
@flags.boolean({ description: 'niconico captures not read yet (their releases get the video title, D34)' })
|
||||
declare nicovideo: boolean
|
||||
|
||||
public async run() {
|
||||
let lastId = 0
|
||||
let queued = 0
|
||||
try {
|
||||
while (!this.limit || queued < this.limit) {
|
||||
// Files never inspected: no fingerprint on their binary and no archive listing.
|
||||
const query = db.from('files as f').leftJoin('file_details as d', 'd.file_id', 'f.id').where('f.id', '>', lastId)
|
||||
// Files from NZBs (spots) have no binary.
|
||||
if (this.sevenZip || this.archives || this.sfv || this.nicovideo) query.leftJoin('binaries as b', 'b.file_id', 'f.id')
|
||||
else query.join('binaries as b', 'b.file_id', 'f.id')
|
||||
if (this.nicovideo) {
|
||||
query.whereNull('d.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
|
||||
} else if (this.sfv) {
|
||||
query.whereNull('d.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
|
||||
} else if (this.archives) {
|
||||
query.whereNull('d.content_type').whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
|
||||
} else if (this.sevenZip) {
|
||||
query.whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
|
||||
} else if (this.archiveNzbs) query.whereRaw(`d.archive::text ~* '\\.nzb"'`)
|
||||
else if (!this.all) query.whereNull('b.md5_16k').whereNull('d.archive')
|
||||
const files = await query
|
||||
.orderBy('f.id')
|
||||
.limit(BATCH)
|
||||
.select('f.id', 'f.filename')
|
||||
if (files.length === 0) break
|
||||
lastId = Number(files.at(-1).id)
|
||||
|
||||
const jobs = files
|
||||
.filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
|
||||
.slice(0, this.limit ? this.limit - queued : undefined)
|
||||
.map((f) => Number(f.id))
|
||||
// Into the inspection backlog (InspectionQueue): targeted backfills go ahead of what's waiting;
|
||||
// niconico captures are many (~190k) and only cosmetic, so behind everything else.
|
||||
const priority = this.nicovideo
|
||||
? INSPECTION_PRIORITY.low
|
||||
: this.archiveNzbs || this.sevenZip || this.archives || this.sfv
|
||||
? INSPECTION_PRIORITY.targeted
|
||||
: INSPECTION_PRIORITY.normal
|
||||
if (jobs.length > 0) await InspectionQueue.add(jobs, priority)
|
||||
queued += jobs.length
|
||||
}
|
||||
this.logger.info(`Queued ${queued} files for content inspection.`)
|
||||
} finally {
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,22 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import IndexMaintenance from '#services/IndexMaintenance'
|
||||
|
||||
/** Rebuilds bloated indexes online (gc:scheduler also does this weekly). */
|
||||
export default class DbReindex extends BaseCommand {
|
||||
public static commandName = 'db:reindex'
|
||||
public static description = 'Rebuilds B-tree indexes that are mostly empty space (online, REINDEX CONCURRENTLY).'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only show which indexes would be rebuilt' })
|
||||
declare dryRun: boolean
|
||||
|
||||
public async run() {
|
||||
const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
|
||||
const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
|
||||
this.logger.info(`${rebuilt.length} indexes ${this.dryRun ? 'would be rebuilt' : `rebuilt, ${Math.round(saved / 1024 / 1024)} MB freed`}.`)
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import DeferredService from '#services/DeferredService'
|
||||
|
||||
/**
|
||||
* Sends recorded deferred ranges (fully obfuscated ngPost articles, CONCEPT.md
|
||||
* D28) to the body queue, oldest first. Every article in them is downloaded:
|
||||
* their headers can't be grouped into files.
|
||||
*/
|
||||
export default class DeferredExpandRanges extends BaseCommand {
|
||||
public static commandName = 'deferred:expand-ranges'
|
||||
public static description = 'Queues the bodies of deferred article ranges (costly: every article is downloaded).'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.number({ description: 'Ranges to expand', default: 10 })
|
||||
declare limit: number
|
||||
|
||||
public async run() {
|
||||
try {
|
||||
const { ranges, articles } = await DeferredService.expandRanges(this.limit)
|
||||
this.logger.info(`Expanded ${ranges} ranges: ${articles} articles sent to the body queue.`)
|
||||
} finally {
|
||||
await NntpService.shutdown()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { unquotedFileName } from '#services/NzbImportService'
|
||||
|
||||
/**
|
||||
* Files named after a whole NZB subject without quotes (spot NZBs), e.g.
|
||||
* `Adventure.part0055.rar - [056/175] (1/1)`: renamed to the filename in it.
|
||||
*/
|
||||
export default class FilesCleanNames extends BaseCommand {
|
||||
public static commandName = 'files:clean-names'
|
||||
public static description = 'Strips subject leftovers (counters, "yEnc") from file names that came from NZBs.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only show what would change' })
|
||||
declare dryRun: boolean
|
||||
|
||||
public async run() {
|
||||
const files = await db
|
||||
.from('files')
|
||||
.whereRaw(`filename ~* '(\\(\\d+/\\d+\\)|yEnc|\\[\\d+/\\d+\\])\\s*$'`)
|
||||
.select('id', 'filename')
|
||||
const changes = files
|
||||
.map((f) => ({ id: Number(f.id), from: f.filename as string, to: unquotedFileName(f.filename) }))
|
||||
.filter((c) => c.to !== c.from)
|
||||
for (const c of changes.slice(0, 5)) this.logger.info(`"${c.from}" → "${c.to}"`)
|
||||
if (!this.dryRun) {
|
||||
for (const c of changes) await db.from('files').where('id', c.id).update({ filename: c.to })
|
||||
}
|
||||
this.logger.info(`${changes.length} of ${files.length} file names ${this.dryRun ? 'would change' : 'cleaned'}.`)
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/**
|
||||
* One-off (LOG.md 2026-10-08 (14)): copies the content types and archive
|
||||
* listings still on `files` into file_details, a partition at a time.
|
||||
* Values written since the deploy (by the new code, into file_details) win:
|
||||
* a file that has details already is skipped.
|
||||
*/
|
||||
export default class FilesMoveDetails extends BaseCommand {
|
||||
public static commandName = 'files:move-details'
|
||||
public static description = "Copies files' content types and archive listings into file_details."
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
public async run() {
|
||||
const { rows: partitions } = await db.rawQuery(
|
||||
`SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass ORDER BY c.relname`
|
||||
)
|
||||
let total = 0
|
||||
for (const { relname } of partitions) {
|
||||
const started = Date.now()
|
||||
const result = await db.rawQuery(
|
||||
`INSERT INTO file_details (file_id, content_type, archive)
|
||||
SELECT id, content_type, archive FROM ${relname} WHERE content_type IS NOT NULL OR archive IS NOT NULL
|
||||
ON CONFLICT (file_id) DO NOTHING`
|
||||
)
|
||||
total += result.rowCount ?? 0
|
||||
this.logger.info(`${relname}: ${result.rowCount} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
}
|
||||
this.logger.info(`${total} files' details copied.`)
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
|
||||
/**
|
||||
* Learns the garbage upload signatures now (gc:scheduler does it every
|
||||
* GARBAGE_LEARN_INTERVAL_MS) and lists them (CONCEPT.md D42). --seed first
|
||||
* fills signature_probes from a sample of the files probed before it
|
||||
* existed, so there is something to learn from right away.
|
||||
*/
|
||||
export default class GarbageLearn extends BaseCommand {
|
||||
public static commandName = 'garbage:learn'
|
||||
public static description = 'Learns which upload signatures are garbage and lists them.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.number({ description: 'First add probed releases from TABLESAMPLE SYSTEM (percent) of files' })
|
||||
declare seed?: number
|
||||
|
||||
public async run() {
|
||||
if (this.seed) {
|
||||
const started = Date.now()
|
||||
const added = await GarbageFilter.seed(this.seed)
|
||||
this.logger.info(`Seeded ${added} probed releases from ${this.seed}% of files in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
}
|
||||
const { signatures, dropped, changed } = await GarbageFilter.learn()
|
||||
for (const line of changed) this.logger.info(`Changed: ${line}`)
|
||||
const rows = await db.from('upload_signatures').orderBy('probed', 'desc').limit(40)
|
||||
for (const row of rows) {
|
||||
const rate = row.probed ? ((100 * row.named) / row.probed).toFixed(1) : '-'
|
||||
this.logger.info(`${row.dropped ? 'DROP' : 'keep'} ${row.tool.padEnd(10)} ${(row.group_name || '(no group)').padEnd(36)} ${String(row.named).padStart(6)}/${String(row.probed).padEnd(6)} named (${rate}%)`)
|
||||
}
|
||||
this.logger.info(`${dropped} of ${signatures} signatures dropped.`)
|
||||
}
|
||||
}
|
||||
@@ -1,424 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import type { Queue } from 'bullmq'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import gcConfig from '#config/gc'
|
||||
import QueueService from '#services/QueueService'
|
||||
import RedisService from '#services/RedisService'
|
||||
import IndexMaintenance from '#services/IndexMaintenance'
|
||||
import ReleaseMergeService from '#services/ReleaseMergeService'
|
||||
import StagingService from '#services/StagingService'
|
||||
import FilePartitions from '#services/FilePartitions'
|
||||
import ReleaseCompleteness from '#services/ReleaseCompleteness'
|
||||
import FileKeys from '#services/FileKeys'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
|
||||
const CLEAN_BATCH = 1000
|
||||
const SCAN_COUNT = 1000
|
||||
/** Binaries per statement when dropping copied parts (a few hundred parts each at most). */
|
||||
const COPIED_PARTS_BATCH = 2000
|
||||
|
||||
export default class GcScheduler extends BaseCommand {
|
||||
public static commandName = 'gc:scheduler'
|
||||
public static description = 'Periodically removes old BullMQ jobs and stale binaries.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Run one sweep and exit' })
|
||||
declare once: boolean
|
||||
|
||||
@flags.boolean({ description: 'Report what would be removed without changing anything' })
|
||||
declare dryRun: boolean
|
||||
|
||||
/** When release completeness was last recomputed (it runs less often than the sweep). */
|
||||
private completenessAt = 0
|
||||
/** When the garbage signatures were last learned (D42). */
|
||||
private learnedAt = 0
|
||||
/** When split uploads were last merged. */
|
||||
private mergedAt = 0
|
||||
|
||||
public async run() {
|
||||
this.logger.info(
|
||||
`GC started (interval ${gcConfig.intervalMs / 1000}s${this.dryRun ? ', dry run' : ''}).`
|
||||
)
|
||||
|
||||
try {
|
||||
do {
|
||||
try {
|
||||
await this.sweep()
|
||||
} catch (err: any) {
|
||||
this.logger.error(`GC sweep failed: ${err.message}`)
|
||||
}
|
||||
try {
|
||||
await this.releaseCompleteness()
|
||||
} catch (err: any) {
|
||||
this.logger.error(`Release completeness failed: ${err.message}`)
|
||||
}
|
||||
try {
|
||||
await this.fillFileKeys()
|
||||
} catch (err: any) {
|
||||
this.logger.error(`file_keys fill failed: ${err.message}`)
|
||||
}
|
||||
try {
|
||||
await this.learnGarbage()
|
||||
} catch (err: any) {
|
||||
this.logger.error(`Garbage signatures failed: ${err.message}`)
|
||||
}
|
||||
if (!this.once) await Shutdown.sleep(gcConfig.intervalMs)
|
||||
} while (!this.once && !Shutdown.requested)
|
||||
} finally {
|
||||
await QueueService.closeAll()
|
||||
await RedisService.close()
|
||||
}
|
||||
}
|
||||
|
||||
private async sweep() {
|
||||
const { completedAgeSeconds, failedAgeSeconds } = gcConfig.jobs
|
||||
for (const queue of [...QueueService.all, ...QueueService.legacy]) {
|
||||
const completed = await this.cleanJobs(queue, 'completed', completedAgeSeconds)
|
||||
const failed = await this.cleanJobs(queue, 'failed', failedAgeSeconds)
|
||||
if (completed || failed) {
|
||||
this.logger.info(
|
||||
`${queue.name}: ${this.verb('remove', 'removed')} ${completed} completed and ${failed} failed jobs.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// Inserts fail without a partition for their id (D39). Four are kept ahead, so a failure here
|
||||
// can wait for the next sweep; it used to end the sweep, deletes and all (LOG.md 2026-10-07 (4)).
|
||||
if (!this.dryRun) {
|
||||
try {
|
||||
await FilePartitions.ensure()
|
||||
} catch (err: any) {
|
||||
this.logger.error(`files partitions: ${err.message}; trying again next sweep.`)
|
||||
}
|
||||
}
|
||||
|
||||
const trimmed = await this.trimBodyQueue()
|
||||
if (trimmed) this.logger.info(`Body queue: ${this.verb('drop', 'dropped')} ${trimmed} oldest waiting jobs.`)
|
||||
|
||||
const binaries = await this.deleteStaleBinaries()
|
||||
this.logger.info(`Binaries: ${this.verb('delete', 'deleted')} ${binaries} stale.`)
|
||||
|
||||
if (!this.dryRun) {
|
||||
// Each kind gets its own time budget; empty releases last, as deleting binaries empties them.
|
||||
const budget = () => Date.now() + gcConfig.deleteBudgetMs
|
||||
const parts = await this.dropCopiedParts(budget())
|
||||
const single = await this.deleteSingleArticleBinaries(budget())
|
||||
const unfingerprinted = await this.deleteUnfingerprintedBinaries(budget())
|
||||
const releases = await this.deleteEmptyReleases(budget())
|
||||
const staging = await StagingService.sweep(gcConfig.stagingTtlSeconds, budget())
|
||||
if (staging.adopted || staging.expired) {
|
||||
this.logger.info(`Staging: ${staging.adopted} segments adopted by their file, ${staging.expired} expired.`)
|
||||
}
|
||||
if (parts || single || unfingerprinted || releases) {
|
||||
this.logger.info(
|
||||
`Space: ${parts} copied parts, ${single} single-article binaries, ${unfingerprinted} unfingerprinted completed binaries, ` +
|
||||
`${releases} empty releases deleted.`
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
const deferred = await this.deleteOldDeferredFiles()
|
||||
if (deferred) this.logger.info(`Deferred files: ${this.verb('delete', 'deleted')} ${deferred} past retention.`)
|
||||
|
||||
const { expired, scanned } = await this.expireLegacyPendingFiles()
|
||||
if (scanned > 0) {
|
||||
this.logger.info(
|
||||
`Legacy pending files: scanned ${scanned}, ${this.verb('set a TTL on', 'set a TTL on')} ${expired} without expiry.`
|
||||
)
|
||||
}
|
||||
|
||||
// Uploads split per poster (CONCEPT.md D35); the releases this empties go in a later sweep. It reads
|
||||
// every release and its files: hourly, not every sweep (LOG.md 2026-10-06 (12)).
|
||||
if (this.once || Date.now() - this.mergedAt >= gcConfig.mergeIntervalMs) {
|
||||
this.mergedAt = Date.now()
|
||||
const merged = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
|
||||
if (merged.releases) this.logger.info(`Split uploads: ${this.verb('merge', 'merged')} ${merged.releases} releases into ${merged.names}.`)
|
||||
}
|
||||
|
||||
await this.rebuildIndexes()
|
||||
}
|
||||
|
||||
/** Rebuilds bloated indexes, at most every gcConfig.reindexIntervalMs (CONCEPT.md D33). */
|
||||
/** The keys of files from before file_keys, a slice of time per sweep, until done. */
|
||||
private async fillFileKeys() {
|
||||
if (this.dryRun || (await FileKeys.complete())) return
|
||||
const started = Date.now()
|
||||
const { read, complete, nextId, untilId } = await FileKeys.fill(gcConfig.fileKeysFillMs)
|
||||
this.logger.info(
|
||||
`file_keys: ${read.toLocaleString('en')} keys copied in ${Math.round((Date.now() - started) / 1000)} s, ` +
|
||||
(complete ? 'complete: file inserts use the filter now.' : `at file ${nextId.toLocaleString('en')} of ${untilId.toLocaleString('en')}.`)
|
||||
)
|
||||
}
|
||||
|
||||
private async releaseCompleteness() {
|
||||
if (this.dryRun || (!this.once && Date.now() - this.completenessAt < gcConfig.releaseCompleteIntervalMs)) return
|
||||
this.completenessAt = Date.now()
|
||||
const started = Date.now()
|
||||
const checked = await ReleaseCompleteness.sweep()
|
||||
if (checked > 0) this.logger.info(`Release completeness: ${checked} changed releases checked in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
}
|
||||
|
||||
/** Learns which upload signatures are garbage from the probed releases (CONCEPT.md D42). */
|
||||
private async learnGarbage() {
|
||||
if (this.dryRun || (!this.once && Date.now() - this.learnedAt < indexerConfig.garbage.learnIntervalMs)) return
|
||||
this.learnedAt = Date.now()
|
||||
const { signatures, dropped, changed } = await GarbageFilter.learn()
|
||||
this.logger.info(`Garbage signatures: ${dropped} of ${signatures} dropped.`)
|
||||
for (const line of changed) this.logger.info(`Garbage signature ${line}`)
|
||||
}
|
||||
|
||||
private async rebuildIndexes() {
|
||||
const key = 'gc:reindex:last'
|
||||
const last = Number((await RedisService.client.get(key)) ?? 0)
|
||||
if (Date.now() - last < gcConfig.reindexIntervalMs) return
|
||||
const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
|
||||
if (!this.dryRun) await RedisService.client.set(key, String(Date.now()))
|
||||
const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
|
||||
this.logger.info(`Indexes: ${this.verb('rebuild', 'rebuilt')} ${rebuilt.length}, ${Math.round(saved / 1024 / 1024)} MB freed.`)
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops the oldest waiting body jobs above gcConfig.bodyQueueMax. BullMQ
|
||||
* workers take jobs from the right end of the wait list, so the oldest are
|
||||
* there.
|
||||
*/
|
||||
private async trimBodyQueue(): Promise<number> {
|
||||
const redis = RedisService.client
|
||||
const key = QueueService.bodyQueue.toKey('wait')
|
||||
const excess = (await redis.llen(key)) - gcConfig.bodyQueueMax
|
||||
if (excess <= 0 || this.dryRun) return Math.max(excess, 0)
|
||||
let dropped = 0
|
||||
while (dropped < excess) {
|
||||
const ids: string[] = (await redis.rpop(key, Math.min(5000, excess - dropped))) ?? []
|
||||
if (ids.length === 0) break
|
||||
await redis.unlink(...ids.map((id) => QueueService.bodyQueue.toKey(id)))
|
||||
dropped += ids.length
|
||||
}
|
||||
return dropped
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes binaries (and, by cascade, their parts) that received no segment
|
||||
* for `staleAfterSeconds`: incomplete ones are given up on; completed ones
|
||||
* already live on in `files`; conflicted ones will never become a file.
|
||||
* Completed ones still waiting for worker:file are kept.
|
||||
*/
|
||||
/**
|
||||
* Binaries without new segments for staleAfterSeconds: incomplete ones are given up on,
|
||||
* finished ones have their file. A finished one whose file waits for content inspection
|
||||
* stays until it's inspected (its fingerprint names it from par2 sets, D16), at most
|
||||
* awaitInspectionSeconds (LOG.md 2026-10-06 (11)). Three deletes, one per partial index,
|
||||
* so none reads past millions of waiting rows.
|
||||
*/
|
||||
private async deleteStaleBinaries(): Promise<number> {
|
||||
const stale = new Date(Date.now() - gcConfig.binaries.staleAfterSeconds * 1000)
|
||||
const waited = new Date(Date.now() - gcConfig.binaries.awaitInspectionSeconds * 1000)
|
||||
const deletes = [
|
||||
db.from('binaries').where('last_seen_at', '<', stale).whereNotNull('file_id').whereRaw('awaiting_inspection IS NOT TRUE'),
|
||||
db.from('binaries').where('last_seen_at', '<', stale).where((q) => q.whereNull('completed_at').orWhereNotNull('conflicted_at')),
|
||||
db.from('binaries').where('last_seen_at', '<', waited).whereRaw('awaiting_inspection IS TRUE'),
|
||||
]
|
||||
let total = 0
|
||||
for (const query of deletes) {
|
||||
if (this.dryRun) total += Number((await query.count('* as count'))[0].count)
|
||||
else total += Number(await query.delete())
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
private async deleteOldDeferredFiles(): Promise<number> {
|
||||
const cutoff = new Date(Date.now() - gcConfig.deferredRetentionDays * 24 * 60 * 60 * 1000)
|
||||
let total = 0
|
||||
for (const [table, column] of [['deferred_files', 'first_seen_at'], ['deferred_ranges', 'created_at']]) {
|
||||
const query = db.from(table).where(column, '<', cutoff)
|
||||
if (this.dryRun) {
|
||||
const [{ count }] = await query.count('* as count')
|
||||
total += Number(count)
|
||||
} else {
|
||||
total += Number(await query.delete())
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// Keyset cursors over the tables, kept between sweeps so one pass over
|
||||
// millions of rows is spread over several sweeps.
|
||||
private singleCursor = 0
|
||||
private releaseCursor = 0
|
||||
|
||||
/**
|
||||
* Header parts of binaries that are a file already. worker:file drops them
|
||||
* when it creates the file, and ingest no longer adds them afterwards; this
|
||||
* catches what slipped through. A cursor over binaries (as for single
|
||||
* articles) visits each one once per pass, instead of searching all of
|
||||
* them for parts on every batch (LOG.md 2026-10-04).
|
||||
*/
|
||||
private copiedCursor = 0
|
||||
private async dropCopiedParts(budget: number): Promise<number> {
|
||||
let total = 0
|
||||
while (Date.now() < budget) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH batch AS (
|
||||
SELECT id FROM binaries WHERE id > ? AND file_id IS NOT NULL ORDER BY id LIMIT ?
|
||||
), gone AS (
|
||||
DELETE FROM parts p USING batch WHERE p.binary_id = batch.id AND p.pcrc32 IS NULL
|
||||
RETURNING 1)
|
||||
SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
|
||||
[this.copiedCursor, COPIED_PARTS_BATCH]
|
||||
)
|
||||
total += Number(rows[0].deleted)
|
||||
if (rows[0].last === null) {
|
||||
this.copiedCursor = 0
|
||||
break
|
||||
}
|
||||
this.copiedCursor = Number(rows[0].last)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
/** Header-derived binaries that never got a second article (gcConfig.binaries.singleArticleAfterSeconds). */
|
||||
private async deleteSingleArticleBinaries(budget: number): Promise<number> {
|
||||
const cutoff = new Date(Date.now() - gcConfig.binaries.singleArticleAfterSeconds * 1000)
|
||||
let total = 0
|
||||
while (Date.now() < budget) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH batch AS (
|
||||
SELECT id FROM binaries WHERE id > ? ORDER BY id LIMIT ?
|
||||
), gone AS (
|
||||
DELETE FROM binaries b USING batch
|
||||
WHERE b.id = batch.id AND b.completed_at IS NULL AND b.size IS NULL AND b.total_parts > 1
|
||||
AND b.last_seen_at < ? AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) <= 1
|
||||
RETURNING 1)
|
||||
SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
|
||||
[this.singleCursor, gcConfig.deleteBatch, cutoff]
|
||||
)
|
||||
total += Number(rows[0].deleted)
|
||||
if (rows[0].last === null) {
|
||||
this.singleCursor = 0
|
||||
break
|
||||
}
|
||||
this.singleCursor = Number(rows[0].last)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
/**
|
||||
* Completed header-derived binaries without a fingerprint, some hours after
|
||||
* completion: their file exists, their parts are gone, and par2 naming needs
|
||||
* a size and MD5 they don't have. Oldest first, via the last_seen_at index.
|
||||
*/
|
||||
private async deleteUnfingerprintedBinaries(budget: number): Promise<number> {
|
||||
const cutoff = new Date(Date.now() - gcConfig.binaries.unfingerprintedAfterSeconds * 1000)
|
||||
let total = 0
|
||||
while (Date.now() < budget) {
|
||||
const result = await db.rawQuery(
|
||||
`DELETE FROM binaries WHERE id IN (
|
||||
SELECT id FROM binaries
|
||||
WHERE last_seen_at < ? AND file_id IS NOT NULL AND completed_at IS NOT NULL
|
||||
AND size IS NULL AND md5_16k IS NULL AND conflicted_at IS NULL
|
||||
AND awaiting_inspection IS NOT TRUE
|
||||
ORDER BY last_seen_at LIMIT ?)`,
|
||||
[cutoff, gcConfig.deleteBatch]
|
||||
)
|
||||
total += result.rowCount
|
||||
if (result.rowCount < gcConfig.deleteBatch) break
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
/** Releases nothing refers to any more: no files, binaries, par2 listing or spot. */
|
||||
private async deleteEmptyReleases(budget: number): Promise<number> {
|
||||
const cutoff = new Date(Date.now() - gcConfig.emptyReleaseAfterSeconds * 1000)
|
||||
let total = 0
|
||||
while (Date.now() < budget) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH batch AS (
|
||||
SELECT id FROM releases WHERE id > ? ORDER BY id LIMIT ?
|
||||
), gone AS (
|
||||
DELETE FROM releases r USING batch
|
||||
WHERE r.id = batch.id AND r.file_count = 0 AND r.first_seen_at < ?
|
||||
AND NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id)
|
||||
RETURNING 1)
|
||||
SELECT (SELECT max(id) FROM batch) AS last, (SELECT count(*) FROM gone) AS deleted`,
|
||||
[this.releaseCursor, gcConfig.deleteBatch, cutoff]
|
||||
)
|
||||
total += Number(rows[0].deleted)
|
||||
if (rows[0].last === null) {
|
||||
this.releaseCursor = 0
|
||||
break
|
||||
}
|
||||
this.releaseCursor = Number(rows[0].last)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
/** Removes jobs in `state` that finished more than `ageSeconds` ago. */
|
||||
private async cleanJobs(queue: Queue, state: 'completed' | 'failed', ageSeconds: number) {
|
||||
const graceMs = ageSeconds * 1000
|
||||
if (this.dryRun) {
|
||||
const cutoff = Date.now() - graceMs
|
||||
// Finished-job sets are sorted by finishedOn, so a score range counts them cheaply.
|
||||
return RedisService.client.zcount(queue.toKey(state), '-inf', cutoff)
|
||||
}
|
||||
|
||||
let total = 0
|
||||
let removed: string[]
|
||||
do {
|
||||
removed = await queue.clean(graceMs, CLEAN_BATCH, state)
|
||||
total += removed.length
|
||||
} while (removed.length === CLEAN_BATCH)
|
||||
return total
|
||||
}
|
||||
|
||||
/**
|
||||
* Pending-file hashes of the old header worker (and the pre-AdonisJS app)
|
||||
* never expire. Give them a TTL rather than deleting them outright.
|
||||
*/
|
||||
private async expireLegacyPendingFiles() {
|
||||
const redis = RedisService.client
|
||||
const { prefix, ttlSeconds } = gcConfig.legacyPendingFiles
|
||||
let cursor = '0'
|
||||
let scanned = 0
|
||||
let expired = 0
|
||||
|
||||
do {
|
||||
const [next, keys]: [string, string[]] = await redis.scan(
|
||||
cursor,
|
||||
'MATCH',
|
||||
`${prefix}*`,
|
||||
'COUNT',
|
||||
SCAN_COUNT
|
||||
)
|
||||
cursor = next
|
||||
if (keys.length === 0) continue
|
||||
scanned += keys.length
|
||||
|
||||
const ttls: [Error | null, number][] = await keys
|
||||
.reduce((pipeline, key) => pipeline.ttl(key), redis.pipeline())
|
||||
.exec()
|
||||
const orphaned = keys.filter((_, i) => ttls[i][1] === -1)
|
||||
expired += orphaned.length
|
||||
|
||||
if (!this.dryRun && orphaned.length > 0) {
|
||||
await orphaned
|
||||
.reduce((pipeline, key) => pipeline.expire(key, ttlSeconds, 'NX'), redis.pipeline())
|
||||
.exec()
|
||||
}
|
||||
} while (cursor !== '0')
|
||||
|
||||
return { scanned, expired }
|
||||
}
|
||||
|
||||
private verb(present: string, past: string) {
|
||||
return this.dryRun ? `would ${present}` : past
|
||||
}
|
||||
}
|
||||
@@ -1,79 +0,0 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import Group from '#models/group'
|
||||
import NntpService from '#services/NntpService'
|
||||
import { parseGroupResponse } from '#lib/nntp/group'
|
||||
|
||||
/**
|
||||
* Adds newsgroups to index (or reactivates them). The scheduler picks them
|
||||
* up on its next tick, starting INDEX_INITIAL_BACKFILL articles back.
|
||||
*/
|
||||
export default class GroupAdd extends BaseCommand {
|
||||
public static commandName = 'group:add'
|
||||
public static description = 'Adds newsgroups to index.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@args.spread({ description: 'Group names, e.g. alt.binaries.hdtv' })
|
||||
declare names: string[]
|
||||
|
||||
@flags.boolean({ description: 'Deactivate the groups instead' })
|
||||
declare disable: boolean
|
||||
|
||||
@flags.boolean({
|
||||
description: 'Also download bodies to identify obfuscated posts (one body per article; bandwidth-heavy)',
|
||||
})
|
||||
declare bodies: boolean
|
||||
|
||||
@flags.number({ description: 'Start this many articles back instead of INDEX_INITIAL_BACKFILL (new groups only)' })
|
||||
declare backfill?: number
|
||||
|
||||
@flags.boolean({ description: 'With --backfill: also for groups indexed before, skipping what was missed since' })
|
||||
declare reset: boolean
|
||||
|
||||
@flags.boolean({ description: "Start at the server's oldest article: the group's full history (also for groups indexed before)" })
|
||||
declare fromStart: boolean
|
||||
|
||||
public async run() {
|
||||
for (const name of this.names) {
|
||||
if (!/^[a-z0-9][a-z0-9.+_-]*$/i.test(name)) {
|
||||
this.logger.error(`Not a valid group name: ${name}`)
|
||||
continue
|
||||
}
|
||||
const group = await Group.firstOrCreate({ name }, { name, active: !this.disable, fetchBodies: this.bodies })
|
||||
group.active = !this.disable
|
||||
if (this.bodies) group.fetchBodies = true
|
||||
if (this.fromStart) {
|
||||
// The scheduler continues after last_queued_id; 0 would mean "new group" to it.
|
||||
const { first } = await this.groupInfo(name)
|
||||
group.lastQueuedId = first > 1n ? first - 1n : 1n
|
||||
group.lastIndexedId = group.lastQueuedId
|
||||
} else if (this.backfill !== undefined && (this.reset || BigInt(group.lastQueuedId ?? 0) === 0n)) {
|
||||
const { last } = await this.groupInfo(name)
|
||||
group.lastQueuedId = (last > BigInt(this.backfill) ? last - BigInt(this.backfill) : 0n)
|
||||
group.lastIndexedId = group.lastQueuedId
|
||||
}
|
||||
await group.save()
|
||||
this.logger.info(
|
||||
`${name}: ${group.active ? 'active' : 'inactive'}, bodies ${group.fetchBodies ? 'on' : 'off'}` +
|
||||
(this.backfill !== undefined || this.fromStart ? `, starting after article ${group.lastQueuedId}` : '')
|
||||
)
|
||||
}
|
||||
if (this.backfill !== undefined || this.fromStart) await NntpService.shutdown()
|
||||
}
|
||||
|
||||
private async groupInfo(name: string) {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
return parseGroupResponse((await conn.group(name)).resp)
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,52 +0,0 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import NntpService from '#services/NntpService'
|
||||
import BackfillService from '#services/BackfillService'
|
||||
import { isSpotGroup } from '#services/SpotService'
|
||||
|
||||
/**
|
||||
* Indexes a group's older headers (CONCEPT.md D32): from where live
|
||||
* indexing began back to N days ago, header-only and behind live indexing.
|
||||
* index:scheduler queues the ranges; this only sets where to go.
|
||||
*/
|
||||
export default class GroupBackfill extends BaseCommand {
|
||||
public static commandName = 'group:backfill'
|
||||
public static description = "Indexes groups' older headers (header-only, behind live indexing)."
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@args.spread({ description: 'Groups (default: every active binary group)', required: false })
|
||||
declare names?: string[]
|
||||
|
||||
@flags.number({ description: 'How many days back', default: 21 })
|
||||
declare days: number
|
||||
|
||||
@flags.boolean({ description: 'Stop backfilling these groups (ranges already queued still run)' })
|
||||
declare stop: boolean
|
||||
|
||||
public async run() {
|
||||
const names = this.names?.length
|
||||
? this.names
|
||||
: (await db.from('groups').where('active', true).orderBy('name').select('name')).map((g) => g.name as string).filter((n) => !isSpotGroup(n))
|
||||
try {
|
||||
for (const name of names) {
|
||||
if (this.stop) {
|
||||
await BackfillService.stop(name)
|
||||
this.logger.info(`${name}: backfill stopped.`)
|
||||
continue
|
||||
}
|
||||
const range = await BackfillService.start(name, this.days)
|
||||
this.logger.info(
|
||||
range
|
||||
? `${name}: backfilling articles ${range.from}–${range.to} (${(range.to - range.from).toLocaleString('en')} articles, ${this.days} days).`
|
||||
: `${name}: nothing to backfill.`
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
await NntpService.shutdown()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,176 +0,0 @@
|
||||
import { writeFile } from 'node:fs/promises'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import NntpService from '#services/NntpService'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import { parseGroupResponse } from '#lib/nntp/group'
|
||||
import { parseSubject } from '#lib/subject/index'
|
||||
|
||||
interface GroupRate {
|
||||
name: string
|
||||
perMinute: number
|
||||
sampled?: number
|
||||
normal?: number
|
||||
obfuscated?: number
|
||||
skipped?: number
|
||||
avgArticleBytes?: number
|
||||
avgHeaderBytes?: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists the busiest newsgroups (by articles posted per minute, from two
|
||||
* LIST ACTIVE snapshots) and optionally samples their newest headers to
|
||||
* estimate what indexing them would cost. See LOG.md (2026-09-28 (14)).
|
||||
*
|
||||
* node ace groups:top --minutes 5 --limit 1000 --sample 300 --out /data/top-groups.txt
|
||||
*/
|
||||
export default class GroupsTop extends BaseCommand {
|
||||
public static commandName = 'groups:top'
|
||||
public static description = 'Lists the busiest binary groups and estimates the bandwidth to index them.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.string({ description: 'Group pattern for LIST ACTIVE', default: 'alt.binaries.*' })
|
||||
declare pattern: string
|
||||
|
||||
@flags.number({ description: 'Minutes between the two snapshots', default: 5 })
|
||||
declare minutes: number
|
||||
|
||||
@flags.number({ description: 'Number of groups to list', default: 1000 })
|
||||
declare limit: number
|
||||
|
||||
@flags.number({ description: 'Headers to sample per listed group (0 = no sampling)', default: 0 })
|
||||
declare sample: number
|
||||
|
||||
@flags.string({ description: 'Write the list to this file (default: print only)' })
|
||||
declare out?: string
|
||||
|
||||
public async run() {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
this.logger.info(`Snapshot 1 of ${this.pattern}…`)
|
||||
const first = await this.snapshot(conn)
|
||||
this.logger.info(`${first.size} groups. Waiting ${this.minutes} min for snapshot 2…`)
|
||||
const started = Date.now()
|
||||
await sleep(this.minutes * 60_000)
|
||||
const second = await this.snapshot(conn)
|
||||
const minutes = (Date.now() - started) / 60_000
|
||||
|
||||
const groups: GroupRate[] = []
|
||||
for (const [name, high] of second) {
|
||||
const before = first.get(name)
|
||||
if (before === undefined || high <= before) continue
|
||||
groups.push({ name, perMinute: Number(high - before) / minutes })
|
||||
}
|
||||
groups.sort((a, b) => b.perMinute - a.perMinute)
|
||||
const top = groups.slice(0, this.limit)
|
||||
this.logger.info(`${groups.length} active groups; listing ${top.length}.`)
|
||||
|
||||
if (this.sample > 0) {
|
||||
for (const [i, group] of top.entries()) {
|
||||
try {
|
||||
Object.assign(group, await this.sampleGroup(conn, group.name))
|
||||
} catch (error: any) {
|
||||
this.logger.warning(`${group.name}: ${error.message}`)
|
||||
}
|
||||
if ((i + 1) % 50 === 0) this.logger.info(`Sampled ${i + 1}/${top.length} groups…`)
|
||||
}
|
||||
}
|
||||
|
||||
const text = this.format(top, minutes)
|
||||
if (this.out) {
|
||||
await writeFile(this.out, text)
|
||||
this.logger.info(`Written to ${this.out}`)
|
||||
} else {
|
||||
console.log(text)
|
||||
}
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
await NntpService.shutdown()
|
||||
}
|
||||
}
|
||||
|
||||
/** Group name → highest article number, from LIST ACTIVE. */
|
||||
private async snapshot(conn: any): Promise<Map<string, bigint>> {
|
||||
const { data } = await conn.list(this.pattern)
|
||||
const result = new Map<string, bigint>()
|
||||
for (const line of Buffer.from(data).toString('latin1').split('\r\n')) {
|
||||
const [name, high] = line.split(' ')
|
||||
if (name && /^\d+$/.test(high ?? '')) result.set(name, BigInt(high))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/** Classifies the newest headers the way worker:fetch does. */
|
||||
private async sampleGroup(conn: any, name: string) {
|
||||
const { last } = parseGroupResponse((await conn.group(name)).resp)
|
||||
const start = last - BigInt(this.sample) + 1n
|
||||
const { overviews } = await conn.xover((start > 0n ? start : 1n).toString(), last.toString())
|
||||
let normal = 0
|
||||
let skipped = 0
|
||||
let obfuscated = 0
|
||||
let obfuscatedBytes = 0
|
||||
let headerBytes = 0
|
||||
for (const [, header] of overviews as [number, Record<string, string>][]) {
|
||||
if (!header) continue
|
||||
headerBytes += Object.values(header).reduce((sum, v) => sum + String(v).length + 1, 0)
|
||||
if (header.subject && parseSubject(header.subject)) normal++
|
||||
else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
|
||||
else {
|
||||
obfuscated++
|
||||
obfuscatedBytes += Number(header[':bytes']) || 0
|
||||
}
|
||||
}
|
||||
const sampled = normal + skipped + obfuscated
|
||||
return {
|
||||
sampled,
|
||||
normal,
|
||||
skipped,
|
||||
obfuscated,
|
||||
avgArticleBytes: obfuscated ? obfuscatedBytes / obfuscated : 0,
|
||||
avgHeaderBytes: sampled ? headerBytes / sampled : 0,
|
||||
}
|
||||
}
|
||||
|
||||
private format(groups: GroupRate[], minutes: number): string {
|
||||
const sampled = groups.some((g) => g.sampled)
|
||||
const lines = [
|
||||
`# Busiest ${this.pattern} groups by articles/min, measured over ${minutes.toFixed(1)} min on ${new Date().toISOString()}`,
|
||||
sampled
|
||||
? '# rank\tgroup\tarticles/min\tobfuscated%\tskipped%\tbody Mbit/s\theader Mbit/s'
|
||||
: '# rank\tgroup\tarticles/min',
|
||||
]
|
||||
let bodyTotal = 0
|
||||
let headerTotal = 0
|
||||
let articlesTotal = 0
|
||||
for (const [i, g] of groups.entries()) {
|
||||
articlesTotal += g.perMinute
|
||||
if (!g.sampled) {
|
||||
lines.push(`${i + 1}\t${g.name}\t${Math.round(g.perMinute)}`)
|
||||
continue
|
||||
}
|
||||
const obfuscatedShare = g.obfuscated! / g.sampled
|
||||
// Bodies: every obfuscated article is downloaded once. Headers: every article.
|
||||
const bodyMbit = (g.perMinute * obfuscatedShare * g.avgArticleBytes! * 8) / 60 / 1e6
|
||||
const headerMbit = (g.perMinute * g.avgHeaderBytes! * 8) / 60 / 1e6
|
||||
bodyTotal += bodyMbit
|
||||
headerTotal += headerMbit
|
||||
lines.push(
|
||||
[i + 1, g.name, Math.round(g.perMinute), (100 * obfuscatedShare).toFixed(1), ((100 * g.skipped!) / g.sampled).toFixed(1), bodyMbit.toFixed(1), headerMbit.toFixed(2)].join('\t')
|
||||
)
|
||||
}
|
||||
lines.push(`# total: ${Math.round(articlesTotal)} articles/min`)
|
||||
if (sampled) {
|
||||
lines.push(`# bandwidth to index like alt.binaries.test (headers + obfuscated bodies): ${(bodyTotal + headerTotal).toFixed(0)} Mbit/s`)
|
||||
lines.push(`# of which bodies ${bodyTotal.toFixed(0)} Mbit/s, headers ${headerTotal.toFixed(1)} Mbit/s (skip list applied)`)
|
||||
}
|
||||
return lines.join('\n') + '\n'
|
||||
}
|
||||
}
|
||||
@@ -1,144 +0,0 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import Group from '#models/group'
|
||||
import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
import FetchRangeService from '#services/FetchRangeService'
|
||||
import SplitService from '#services/SplitService'
|
||||
import BackfillService from '#services/BackfillService'
|
||||
|
||||
export default class IndexScheduler extends BaseCommand {
|
||||
public static commandName = 'index:scheduler'
|
||||
public static description = 'Periodically checks for new articles and schedules them for fetching.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
private pool = NntpService
|
||||
|
||||
public async run() {
|
||||
this.logger.info('Scheduler started. Awaiting tasks...')
|
||||
|
||||
const schedule = async () => {
|
||||
this.logger.info('Checking for new headers...')
|
||||
const groups = await Group.query().where('active', true)
|
||||
|
||||
if (groups.length === 0) {
|
||||
this.logger.info('No active groups to index. Add some via `node ace db:seed` or manually.')
|
||||
return
|
||||
}
|
||||
|
||||
let conn
|
||||
let failed = false
|
||||
try {
|
||||
conn = await this.pool.acquire()
|
||||
|
||||
for (const group of groups) {
|
||||
try {
|
||||
const { first: firstArticle, last: lastArticle } = await this.pool.selectGroup(
|
||||
conn,
|
||||
group.name
|
||||
)
|
||||
|
||||
await FetchRangeService.requeue(group)
|
||||
|
||||
// Postgres returns bigint columns as strings.
|
||||
const lastQueued = BigInt(group.lastQueuedId ?? 0)
|
||||
const { batchSize, maxBatchesPerTick, initialBackfill } = indexerConfig
|
||||
let startId =
|
||||
lastQueued === 0n ? lastArticle - initialBackfill + 1n : lastQueued + 1n
|
||||
if (startId < firstArticle) startId = firstArticle
|
||||
|
||||
if (startId > lastArticle) {
|
||||
await FetchRangeService.advanceWatermark(group)
|
||||
this.logger.info(`No new headers for group ${group.name}.`)
|
||||
continue
|
||||
}
|
||||
|
||||
let batches = 0
|
||||
let endId = startId - 1n
|
||||
while (endId < lastArticle && batches < maxBatchesPerTick) {
|
||||
const batchStart = endId + 1n
|
||||
endId = batchStart + batchSize - 1n < lastArticle ? batchStart + batchSize - 1n : lastArticle
|
||||
await FetchRangeService.queue(group, batchStart, endId)
|
||||
batches++
|
||||
}
|
||||
|
||||
this.logger.info(
|
||||
`Queued ${batches} fetch job(s) for ${group.name}: articles ${startId} to ${endId}` +
|
||||
(endId < lastArticle ? ` (${lastArticle - endId} left for next tick)` : '')
|
||||
)
|
||||
|
||||
group.lastQueuedId = endId
|
||||
await group.save()
|
||||
const watermark = await FetchRangeService.advanceWatermark(group)
|
||||
if (endId - watermark > batchSize * BigInt(maxBatchesPerTick) * 3n) {
|
||||
this.logger.warning(
|
||||
`${group.name}: indexed up to ${watermark}, ${endId - watermark} articles behind the queue.`
|
||||
)
|
||||
}
|
||||
} catch (err: any) {
|
||||
failed = true
|
||||
this.logger.error(`Error processing group ${group.name}: ${err.message}`)
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
failed = true
|
||||
this.logger.error(`Error in scheduler main loop: ${err.message}`)
|
||||
} finally {
|
||||
if (conn) {
|
||||
this.pool.release(conn, failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const split = async () => {
|
||||
try {
|
||||
const result = await SplitService.splitPending()
|
||||
if (result.parents > 0) {
|
||||
this.logger.info(
|
||||
`Split ${result.parents} conflicted binaries: ${result.volumes} verified volumes, ${result.completed} completed.`
|
||||
)
|
||||
}
|
||||
} catch (err: any) {
|
||||
this.logger.error(`Error splitting conflicted binaries: ${err.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
// The inspection backlog lives in Postgres; BullMQ gets a working set (LOG.md 2026-10-07 (8)).
|
||||
const feedInspection = async () => {
|
||||
const moved = await InspectionQueue.feed()
|
||||
if (moved > 0) this.logger.info(`Inspection: ${moved} files moved from the backlog to the queue.`)
|
||||
}
|
||||
|
||||
const backfill = async () => {
|
||||
try {
|
||||
const queued = await BackfillService.tick()
|
||||
if (queued > 0) this.logger.info(`Queued ${queued} backfill range(s).`)
|
||||
} catch (err: any) {
|
||||
this.logger.error(`Error queuing backfill: ${err.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Run immediately, then wait a full interval after each run so runs never overlap. A failed
|
||||
// round (e.g. Postgres restarting) is logged and the next one tries again: an error that escaped
|
||||
// here once ended the loop while the process stayed up, and live indexing stopped for ~3 hours
|
||||
// (LOG.md 2026-10-07 (3)).
|
||||
while (!Shutdown.requested) {
|
||||
for (const step of [schedule, backfill, split, feedInspection]) {
|
||||
try {
|
||||
await step()
|
||||
} catch (err: any) {
|
||||
this.logger.error(`Scheduler round failed, retrying next round: ${err.message}`)
|
||||
}
|
||||
}
|
||||
await Shutdown.sleep(indexerConfig.intervalMs)
|
||||
}
|
||||
await this.pool.shutdown()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
@@ -1,56 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import QueueService from '#services/QueueService'
|
||||
import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
|
||||
|
||||
/**
|
||||
* One-off (LOG.md 2026-10-07 (8)): moves the inspection jobs waiting in
|
||||
* BullMQ into the inspection_queue table, then empties BullMQ's waiting and
|
||||
* prioritized jobs so Redis gives the memory back. The file id is read from
|
||||
* the job id (file-<id>, content-<id>-<time>), so no job is loaded.
|
||||
* index-scheduler then feeds a working set back in.
|
||||
*/
|
||||
export default class InspectionFromRedis extends BaseCommand {
|
||||
public static commandName = 'inspection:from-redis'
|
||||
public static description = 'Moves waiting inspection jobs from Redis into the inspection_queue table.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only count, change nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
public async run() {
|
||||
const queue = QueueService.collectionQueue
|
||||
const client = await queue.client
|
||||
const prefix = `${queue.opts.prefix ?? 'bull'}:${queue.name}`
|
||||
try {
|
||||
const idOf = (jobId: string) => {
|
||||
const m = /^(?:file|content)-(\d+)/.exec(jobId)
|
||||
return m ? Number(m[1]) : null
|
||||
}
|
||||
let moved = 0
|
||||
let skipped = 0
|
||||
const take = async (ids: string[], priority: number) => {
|
||||
const fileIds = ids.map(idOf).filter((id): id is number => id !== null)
|
||||
skipped += ids.length - fileIds.length
|
||||
if (!this.dryRun) await InspectionQueue.add(fileIds, priority)
|
||||
moved += fileIds.length
|
||||
}
|
||||
const waiting = await client.llen(`${prefix}:wait`)
|
||||
for (let i = 0; i < waiting; i += 50_000) await take(await client.lrange(`${prefix}:wait`, i, i + 49_999), INSPECTION_PRIORITY.normal)
|
||||
// Prioritized jobs were the low-priority niconico captures.
|
||||
const prioritized = await client.zcard(`${prefix}:prioritized`)
|
||||
for (let i = 0; i < prioritized; i += 50_000) await take(await client.zrange(`${prefix}:prioritized`, i, i + 49_999), INSPECTION_PRIORITY.low)
|
||||
this.logger.info(`${this.dryRun ? 'Would move' : 'Moved'} ${moved} jobs into inspection_queue (${skipped} with other ids skipped).`)
|
||||
if (this.dryRun) return
|
||||
// Waiting and prioritized jobs only; active ones finish as they are.
|
||||
await queue.drain()
|
||||
const counts = await queue.getJobCounts('wait', 'prioritized', 'active')
|
||||
this.logger.info(`BullMQ now: ${counts.wait} waiting, ${counts.prioritized} prioritized, ${counts.active} active; table: ${await InspectionQueue.size()}.`)
|
||||
} finally {
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,21 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import ReleaseMergeService from '#services/ReleaseMergeService'
|
||||
|
||||
/** Merges uploads split per poster (gc:scheduler also does this every sweep). */
|
||||
export default class ReleaseMergeSplit extends BaseCommand {
|
||||
public static commandName = 'release:merge-split'
|
||||
public static description = 'Merges releases of one upload that was posted under a different poster per file.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only count them' })
|
||||
declare dryRun: boolean
|
||||
|
||||
public async run() {
|
||||
const { names, releases } = await ReleaseMergeService.mergeSplitPosts({ dryRun: this.dryRun })
|
||||
this.logger.info(`${names} split uploads; ${releases} releases ${this.dryRun ? 'would be' : ''} merged into their first.`)
|
||||
}
|
||||
}
|
||||
@@ -1,165 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import IngestService from '#services/IngestService'
|
||||
import NameService from '#services/NameService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import { rarReleaseIdentity, releaseIdentity } from '#lib/release/index'
|
||||
|
||||
const BATCH = 5000
|
||||
|
||||
/**
|
||||
* Recomputes the release of every binary (and its file) with the current
|
||||
* rules in lib/release. Used after changing those rules.
|
||||
*/
|
||||
export default class ReleaseRebuild extends BaseCommand {
|
||||
public static commandName = 'release:rebuild'
|
||||
public static description = 'Reassigns all binaries and files to releases using the current grouping rules.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({
|
||||
description:
|
||||
'Also drop everything derived from file contents (par2 lists, par2/RAR groupings, real names), ' +
|
||||
'to rebuild it with content:backfill --all',
|
||||
})
|
||||
declare resetContent: boolean
|
||||
|
||||
public async run() {
|
||||
if (this.resetContent) await this.dropContentDerived()
|
||||
|
||||
let lastId = 0
|
||||
let assigned = 0
|
||||
let unassigned = 0
|
||||
try {
|
||||
while (true) {
|
||||
const binaries = await db
|
||||
.from('binaries')
|
||||
.where('id', '>', lastId)
|
||||
.orderBy('id')
|
||||
.limit(BATCH)
|
||||
.select('id', 'poster', 'subject', 'name', 'groups', 'posted_at')
|
||||
if (binaries.length === 0) break
|
||||
lastId = Number(binaries.at(-1).id)
|
||||
|
||||
const segments = binaries.map((binary) => {
|
||||
// Body-derived binaries store a synthetic subject, see segmentFromYenc.
|
||||
const weak = /^"[^"]*" yEnc$/.test(binary.subject)
|
||||
return {
|
||||
id: Number(binary.id),
|
||||
poster: binary.poster,
|
||||
groups: binary.groups,
|
||||
postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
|
||||
weakIdentity: weak,
|
||||
release: releaseIdentity({ poster: binary.poster, subject: binary.subject, name: binary.name, weak }),
|
||||
}
|
||||
})
|
||||
|
||||
await db.transaction(async (trx) => {
|
||||
const releaseIds = await IngestService.upsertReleases(trx.knexClient, segments)
|
||||
const ids = segments.map((s) => s.id)
|
||||
const releaseIdsPerBinary = segments.map((s) =>
|
||||
s.release ? releaseIds.get(s.release.key)! : null
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET release_id = v.release_id
|
||||
FROM unnest(?::bigint[], ?::bigint[]) AS v(id, release_id)
|
||||
WHERE b.id = v.id AND b.release_id IS DISTINCT FROM v.release_id
|
||||
AND NOT (v.release_id IS NULL
|
||||
AND b.release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))
|
||||
-- A posted NZB decided this grouping (NzbImportService); name rules don't override it.
|
||||
AND NOT EXISTS (SELECT 1 FROM files n JOIN file_details nd ON nd.file_id = n.id WHERE n.release_id = b.release_id AND nd.content_type = 'nzb')`,
|
||||
[ids, releaseIdsPerBinary] as any
|
||||
)
|
||||
})
|
||||
assigned += segments.filter((s) => s.release).length
|
||||
unassigned += segments.filter((s) => !s.release).length
|
||||
}
|
||||
|
||||
// Files grouped by their RAR headers: regroup from the stored listing (no downloads).
|
||||
let rarGrouped = 0
|
||||
let rarCursor = 0
|
||||
while (true) {
|
||||
const rows = await db
|
||||
.from('files as f')
|
||||
.join('binaries as b', 'b.file_id', 'f.id')
|
||||
.join('file_details as d', 'd.file_id', 'f.id')
|
||||
.whereNotNull('d.archive')
|
||||
.whereNull('b.release_id')
|
||||
.where('f.id', '>', rarCursor)
|
||||
.orderBy('f.id')
|
||||
.limit(BATCH)
|
||||
.select('f.id as file_id', 'd.archive', 'b.id as binary_id', 'b.poster', 'b.groups', 'b.posted_at')
|
||||
if (rows.length === 0) break
|
||||
rarCursor = Number(rows.at(-1).file_id)
|
||||
const segments = rows.map((r) => ({
|
||||
fileId: Number(r.file_id),
|
||||
binaryId: Number(r.binary_id),
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
postedAt: r.posted_at ? new Date(r.posted_at) : undefined,
|
||||
weakIdentity: true,
|
||||
release: rarReleaseIdentity(r.archive?.entries ?? []),
|
||||
}))
|
||||
const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), segments)
|
||||
for (const s of segments.filter((x) => x.release)) {
|
||||
const releaseId = ids.get(s.release!.key)
|
||||
await db.from('binaries').where('id', s.binaryId).update({ release_id: releaseId })
|
||||
await db.from('files').where('id', s.fileId).update({ release_id: releaseId })
|
||||
rarGrouped++
|
||||
}
|
||||
}
|
||||
|
||||
// Real names from par2 for binaries with a known size (no downloads).
|
||||
let named = 0
|
||||
let nameCursor = 0
|
||||
while (true) {
|
||||
const rows = await db.from('binaries').whereNotNull('size').where('id', '>', nameCursor).orderBy('id').limit(BATCH).select('id')
|
||||
if (rows.length === 0) break
|
||||
nameCursor = Number(rows.at(-1).id)
|
||||
named += await NameService.resolve(rows.map((r) => Number(r.id)))
|
||||
}
|
||||
|
||||
const files = await db.rawQuery(
|
||||
`UPDATE files f SET release_id = b.release_id FROM binaries b
|
||||
WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id`
|
||||
)
|
||||
const orphans = await db.rawQuery(
|
||||
`DELETE FROM releases r WHERE NOT EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM files f WHERE f.release_id = r.id)`
|
||||
)
|
||||
if (this.resetContent) {
|
||||
this.logger.info('Content-derived data dropped; run `node ace content:backfill --all` to rebuild it.')
|
||||
}
|
||||
this.logger.info(
|
||||
`Binaries: ${assigned} in a release, ${unassigned} without, ${rarGrouped} regrouped from RAR headers, ` +
|
||||
`${named} named from par2. ` +
|
||||
`Files updated: ${files.rowCount}. Empty releases removed: ${orphans.rowCount}.`
|
||||
)
|
||||
} finally {
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
|
||||
/** par2 lists, par2/RAR release assignments and recovered names. */
|
||||
private async dropContentDerived() {
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.rawQuery('DELETE FROM release_files')
|
||||
await trx.rawQuery(`UPDATE binaries SET real_name = NULL WHERE real_name IS NOT NULL`)
|
||||
await trx.rawQuery(`UPDATE files SET real_name = NULL WHERE real_name IS NOT NULL`)
|
||||
await trx.rawQuery(`UPDATE file_details SET archive = NULL WHERE archive IS NOT NULL`)
|
||||
await trx.rawQuery(`UPDATE releases SET contents = NULL, encrypted = NULL WHERE contents IS NOT NULL OR encrypted IS NOT NULL`)
|
||||
// Binaries whose name rules find no release were grouped from contents (or wrongly by size).
|
||||
await trx.rawQuery(`
|
||||
UPDATE binaries SET release_id = NULL
|
||||
WHERE subject ~ '^"[^"]*" yEnc$' AND release_id IS NOT NULL
|
||||
AND (name !~ '\\.[A-Za-z0-9]{1,5}$' OR release_id IN (SELECT id FROM releases WHERE source IN ('par2', 'rar')))`)
|
||||
await trx.rawQuery(`
|
||||
UPDATE files f SET release_id = b.release_id FROM binaries b
|
||||
WHERE b.file_id = f.id AND f.release_id IS DISTINCT FROM b.release_id`)
|
||||
})
|
||||
this.logger.info('Dropped par2 lists, content-derived groupings and recovered names.')
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { renameFromStoredContents, renameHiddenNamed } from '#services/ArchiveService'
|
||||
import { regroupAndRenameFromPar2 } from '#services/NameService'
|
||||
|
||||
/** Renames random-named releases from their stored archive listings (no downloads). */
|
||||
export default class ReleaseRename extends BaseCommand {
|
||||
public static commandName = 'release:rename'
|
||||
public static description = 'Renames random-named releases from their par2 listings and archive contents.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only releases named after a hidden archive entry (e.g. a decoy ".x.mkv")' })
|
||||
declare hidden: boolean
|
||||
|
||||
public async run() {
|
||||
if (this.hidden) {
|
||||
this.logger.info(`Renamed ${await renameHiddenNamed()} releases named after a hidden archive entry.`)
|
||||
return
|
||||
}
|
||||
const par2 = await regroupAndRenameFromPar2()
|
||||
this.logger.info(`par2: ${par2.regrouped} files moved into the set that names them, ${par2.renamed} releases renamed.`)
|
||||
this.logger.info(`Renamed ${await renameFromStoredContents()} releases from their archive contents.`)
|
||||
}
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Prints how well files are being grouped into releases. */
|
||||
export default class ReleaseStats extends BaseCommand {
|
||||
public static commandName = 'release:stats'
|
||||
public static description = 'Shows release recognition statistics.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
public async run() {
|
||||
const { rows } = await db.rawQuery(`
|
||||
WITH r AS (
|
||||
SELECT r.id, r.file_total, r.poster IS NULL AS weak, r.source,
|
||||
count(b.id) AS binaries,
|
||||
count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed
|
||||
FROM releases r LEFT JOIN binaries b ON b.release_id = r.id
|
||||
GROUP BY r.id
|
||||
)
|
||||
SELECT
|
||||
count(*) AS releases,
|
||||
count(*) FILTER (WHERE completed > 0) AS with_files,
|
||||
count(*) FILTER (WHERE completed >= 2) AS multi_file,
|
||||
count(*) FILTER (WHERE file_total IS NOT NULL AND completed >= file_total) AS complete_by_counter,
|
||||
count(*) FILTER (WHERE weak) AS from_obfuscated,
|
||||
count(*) FILTER (WHERE weak AND completed > 0) AS from_obfuscated_with_files
|
||||
FROM r`)
|
||||
const { rows: coverage } = await db.rawQuery(`
|
||||
SELECT count(*) AS completed_files,
|
||||
count(*) FILTER (WHERE release_id IS NOT NULL) AS in_release
|
||||
FROM binaries WHERE completed_at IS NOT NULL`)
|
||||
const { rows: bySource } = await db.rawQuery(`
|
||||
SELECT coalesce(r.source, 'unknown') AS source, count(DISTINCT r.id) AS releases,
|
||||
count(b.id) FILTER (WHERE b.completed_at IS NOT NULL) AS completed_files
|
||||
FROM releases r JOIN binaries b ON b.release_id = r.id GROUP BY 1 ORDER BY 2 DESC`)
|
||||
// Readable names: descriptive as posted, or recovered from contents.
|
||||
const { rows: naming } = await db.rawQuery(`
|
||||
WITH f AS (
|
||||
SELECT real_name IS NOT NULL AS recovered,
|
||||
is_readable_name(coalesce(real_name, filename)) AS readable_name,
|
||||
regexp_replace(coalesce(real_name, filename),
|
||||
'(\\.(part\\d+\\.rar|r\\d{2,3}|rar|vol\\d+[+-]\\d+\\.par2|par2|7z(\\.\\d{3})?|zip|\\d{3}|nfo|sfv|nzb|tar|zst|mkv|mp4|avi|flac|mp3|jpg))+$', '', 'i') AS stem,
|
||||
coalesce(real_name, filename) AS name
|
||||
FROM files
|
||||
)
|
||||
SELECT count(*) AS files,
|
||||
count(*) FILTER (WHERE recovered) AS names_recovered,
|
||||
count(*) FILTER (WHERE name ~ '\\.[A-Za-z0-9]{1,5}$' AND readable_name) AS readable
|
||||
FROM f`)
|
||||
this.logger.info(JSON.stringify({ ...rows[0], ...coverage[0] }))
|
||||
this.logger.info(`naming: ${JSON.stringify(naming[0])}`)
|
||||
this.logger.info(`by source: ${JSON.stringify(bySource)}`)
|
||||
}
|
||||
}
|
||||
@@ -1,461 +0,0 @@
|
||||
import { gzipSync, unzipSync } from 'node:zlib'
|
||||
import { spawn, type ChildProcess } from 'node:child_process'
|
||||
import { availableParallelism } from 'node:os'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import nntpConfig from '#config/nntp'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import { XoverClient, parseOverviews, type Overview } from '#lib/nntp/xover_client'
|
||||
import { runRange, type OverviewSource } from '#lib/seed/runner'
|
||||
import type { SeedBatch } from '#lib/seed/wire'
|
||||
import type { OverviewHeader } from '#lib/ingest/segment'
|
||||
import { probeInspection, probeWalk, type ContentTask } from '#lib/content/probe'
|
||||
|
||||
class LeaseLost extends Error {}
|
||||
class Stopping extends Error {}
|
||||
|
||||
interface InspectJob {
|
||||
queue: string
|
||||
jobId: string
|
||||
token: string
|
||||
task: ContentTask
|
||||
}
|
||||
|
||||
interface Lease {
|
||||
id: number
|
||||
group: string
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
/**
|
||||
* A seeding node (CONCEPT.md D38): leases article ranges from the seed API
|
||||
* over HTTPS, fetches and aggregates their headers over its own NNTP
|
||||
* connection(s) (D37), and posts the results back in gzip-compressed
|
||||
* batches. Outgoing requests only: no ports, database or Redis needed.
|
||||
*
|
||||
* SEED_URL=https://seed.example SEED_TOKEN=sn_… node ace seed:node --ranges 4 --inspectors 2
|
||||
*/
|
||||
export default class SeedNode extends BaseCommand {
|
||||
public static commandName = 'seed:node'
|
||||
public static description =
|
||||
'Runs a seeding node: leases ranges from the seed API and submits what it aggregated.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: false,
|
||||
staysAlive: true,
|
||||
}
|
||||
|
||||
@flags.string({ description: 'Seed API base URL (default: SEED_URL)' })
|
||||
declare url?: string
|
||||
|
||||
@flags.string({ description: 'Node token (default: SEED_TOKEN)' })
|
||||
declare token?: string
|
||||
|
||||
@flags.number({
|
||||
description:
|
||||
'Ranges seeded at once (with SEED_FETCH=direct, one NNTP connection each); 0 for an inspection-only node (default: SEED_RANGES or 1)',
|
||||
default: process.env.SEED_RANGES ? Number(process.env.SEED_RANGES) : 1,
|
||||
})
|
||||
declare ranges: number
|
||||
|
||||
@flags.number({
|
||||
description:
|
||||
'Content inspections and archive walks worked on at once, articles through the seed API (default: SEED_INSPECTORS or 0)',
|
||||
default: Number(process.env.SEED_INSPECTORS) || 0,
|
||||
})
|
||||
declare inspectors: number
|
||||
|
||||
@flags.number({ description: 'Articles per XOVER', default: 20_000 })
|
||||
declare chunk: number
|
||||
|
||||
@flags.number({ description: 'XOVER chunks per batch posted to the API', default: 5 })
|
||||
declare submitEvery: number
|
||||
|
||||
@flags.string({
|
||||
description:
|
||||
'relay: headers come through the seed API (no NNTP account needed); direct: own NNTP connections (default: SEED_FETCH or relay)',
|
||||
default: process.env.SEED_FETCH || 'relay',
|
||||
})
|
||||
declare fetch: string
|
||||
|
||||
@flags.number({
|
||||
description:
|
||||
'Processes the ranges and inspections are spread over; one process uses one core (default: SEED_PROCESSES or the number of cores)',
|
||||
default: process.env.SEED_PROCESSES ? Number(process.env.SEED_PROCESSES) : availableParallelism(),
|
||||
})
|
||||
declare processes: number
|
||||
|
||||
@flags.boolean({ description: 'Stop when there is nothing to lease instead of waiting' })
|
||||
declare once: boolean
|
||||
|
||||
private stopping = false
|
||||
/** In a worker process: its number, in front of its log lines. */
|
||||
private readonly child = process.env.SEED_PROCESS ?? ''
|
||||
|
||||
public async run() {
|
||||
const base = (this.url ?? process.env.SEED_URL ?? '').replace(/\/+$/, '')
|
||||
const token = this.token ?? process.env.SEED_TOKEN ?? ''
|
||||
if (!base || !token) {
|
||||
this.logger.error('Set --url and --token (or SEED_URL and SEED_TOKEN).')
|
||||
this.exitCode = 1
|
||||
return
|
||||
}
|
||||
// JavaScript runs on one core per process: several slots are spread over worker processes.
|
||||
const processes = Math.max(1, Math.min(this.processes || 1, this.ranges + this.inspectors))
|
||||
if (!this.child && processes > 1) return this.supervise(base, token, processes)
|
||||
|
||||
const stop = () => {
|
||||
if (!this.stopping && !this.child) this.logger.info('Stopping: giving back the ranges in progress…')
|
||||
this.stopping = true
|
||||
}
|
||||
process.on('SIGTERM', stop)
|
||||
process.on('SIGINT', stop)
|
||||
|
||||
const api = new SeedApi(base, token, (message) => this.logger.warning(message))
|
||||
if (!this.child) this.logger.info(
|
||||
`Seeding node for ${base}: ${this.ranges} range(s) at once, ${this.fetch === 'direct' ? 'own NNTP connections' : 'headers through the seed API'}` +
|
||||
(this.inspectors > 0 ? `; ${this.inspectors} inspection(s) at once, articles through the seed API.` : '.')
|
||||
)
|
||||
await Promise.all([
|
||||
...Array.from({ length: this.ranges }, (_, i) => this.work(api, i + 1)),
|
||||
...Array.from({ length: this.inspectors }, (_, i) => this.inspect(api, i + 1)),
|
||||
])
|
||||
if (!this.child) this.logger.info('Seeding node stopped.')
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs the slots in `processes` worker processes (this command again, with
|
||||
* SEED_PROCESS set), each with its share of the ranges and inspections. A
|
||||
* worker that exits while the node runs is restarted; SIGTERM/SIGINT are
|
||||
* passed on, and the node stops when all workers have given their work back.
|
||||
*/
|
||||
private async supervise(base: string, token: string, processes: number) {
|
||||
// Slots dealt out in turn (ranges first, then inspections), so every process has work.
|
||||
const share = (from: number, to: number, i: number) =>
|
||||
Array.from({ length: to - from }, (_, k) => from + k).filter((k) => k % processes === i).length
|
||||
this.logger.info(
|
||||
`Seeding node for ${base}: ${this.ranges} range(s) and ${this.inspectors} inspection(s) at once, over ${processes} processes, ` +
|
||||
`${this.fetch === 'direct' ? 'own NNTP connections' : 'articles and headers through the seed API'}.`
|
||||
)
|
||||
const children = new Set<ChildProcess>()
|
||||
const start = (n: number): Promise<void> =>
|
||||
new Promise((resolve) => {
|
||||
const args = [
|
||||
...process.execArgv,
|
||||
process.argv[1],
|
||||
'seed:node',
|
||||
`--ranges=${share(0, this.ranges, n - 1)}`,
|
||||
`--inspectors=${share(this.ranges, this.ranges + this.inspectors, n - 1)}`,
|
||||
`--fetch=${this.fetch}`,
|
||||
`--chunk=${this.chunk}`,
|
||||
`--submit-every=${this.submitEvery}`,
|
||||
...(this.once ? ['--once'] : []),
|
||||
]
|
||||
const child = spawn(process.execPath, args, {
|
||||
env: { ...process.env, SEED_URL: base, SEED_TOKEN: token, SEED_PROCESS: String(n) },
|
||||
stdio: 'inherit',
|
||||
})
|
||||
children.add(child)
|
||||
child.on('exit', (code, signal) => {
|
||||
children.delete(child)
|
||||
if (this.stopping || this.once || code === 0) return resolve()
|
||||
this.logger.warning(`Process ${n} exited (${signal ?? code}); restarting in 10 s`)
|
||||
setTimeout(() => start(n).then(resolve), 10_000)
|
||||
})
|
||||
})
|
||||
const stop = (signal: NodeJS.Signals) => {
|
||||
if (!this.stopping) this.logger.info('Stopping: giving back the ranges and inspections in progress…')
|
||||
this.stopping = true
|
||||
for (const child of children) child.kill(signal)
|
||||
}
|
||||
process.on('SIGTERM', () => stop('SIGTERM'))
|
||||
process.on('SIGINT', () => stop('SIGINT'))
|
||||
await Promise.all(Array.from({ length: processes }, (_, i) => start(i + 1)))
|
||||
this.logger.info('Seeding node stopped.')
|
||||
}
|
||||
|
||||
private async work(api: SeedApi, worker: number) {
|
||||
const log = (message: string) => this.logger.info(`[${this.child && `${this.child}/`}${worker}] ${message}`)
|
||||
// The NNTP connection is opened for a lease and kept while leases keep coming; with
|
||||
// nothing to lease it's closed, so an idle node holds no connections.
|
||||
let client: XoverClient | undefined
|
||||
try {
|
||||
while (!this.stopping) {
|
||||
const lease = await this.lease(api, log)
|
||||
if (!lease) {
|
||||
client?.close()
|
||||
client = undefined
|
||||
if (this.once) return
|
||||
await this.pause(60_000)
|
||||
continue
|
||||
}
|
||||
if (this.fetch !== 'direct') {
|
||||
await this.workRange(api, new RelaySource(api, lease.id), lease, log)
|
||||
continue
|
||||
}
|
||||
if (!client) {
|
||||
try {
|
||||
client = await this.connect()
|
||||
} catch (error: any) {
|
||||
// Never started: the lease doesn't count as one of the range's attempts.
|
||||
const ipLimit = /simultaneous IP/i.test(error.message)
|
||||
log(`NNTP login failed: ${error.message}; trying again in ${ipLimit ? 5 : 1} min`)
|
||||
await api
|
||||
.post(`/api/v1/seed/ranges/${lease.id}/release`, {
|
||||
error: `NNTP login: ${error.message}`,
|
||||
uncounted: true,
|
||||
})
|
||||
.catch(() => {})
|
||||
await this.pause(ipLimit ? 300_000 : 60_000)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if (!(await this.workRange(api, client, lease, log))) {
|
||||
client.close()
|
||||
client = undefined
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
client?.close()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Content inspection and archive walks (CONCEPT.md D41): lease a few jobs,
|
||||
* probe each with articles through the seed API, post the results. The
|
||||
* indexer applies them; a job left unanswered goes back after 10 minutes.
|
||||
*/
|
||||
private async inspect(api: SeedApi, slot: number) {
|
||||
const log = (message: string) => this.logger.info(`[inspect ${this.child && `${this.child}/`}${slot}] ${message}`)
|
||||
const fetchArticle = (id: string) => api.get(`/api/v1/seed/articles?id=${encodeURIComponent(id)}`, 4)
|
||||
while (!this.stopping) {
|
||||
let jobs: InspectJob[] = []
|
||||
try {
|
||||
jobs = ((await api.post('/api/v1/seed/inspect/lease', { max: 2 })) as { jobs: InspectJob[] } | null)?.jobs ?? []
|
||||
} catch (error: any) {
|
||||
if (error instanceof Unauthorized) throw error
|
||||
log(`Lease failed: ${error.message}`)
|
||||
}
|
||||
if (jobs.length === 0) {
|
||||
if (this.once) return
|
||||
await this.pause(30_000)
|
||||
continue
|
||||
}
|
||||
for (const job of jobs) {
|
||||
const ref = { queue: job.queue, jobId: job.jobId, token: job.token }
|
||||
if (this.stopping) {
|
||||
await api.post('/api/v1/seed/inspect/release', ref).catch(() => {})
|
||||
continue
|
||||
}
|
||||
try {
|
||||
const result =
|
||||
job.task.kind === 'inspect'
|
||||
? await probeInspection(job.task, fetchArticle, (message) => log(message))
|
||||
: await probeWalk(job.task, fetchArticle)
|
||||
await api.post('/api/v1/seed/inspect/done', { ...ref, result: JSON.stringify(result) })
|
||||
} catch (error: any) {
|
||||
if (error instanceof Unauthorized) throw error
|
||||
// The job's lock expired and it went to another node: nothing to report.
|
||||
if (error instanceof LeaseLost) continue
|
||||
log(`${job.task.kind === 'inspect' ? `"${job.task.name}"` : `walk of release ${job.task.releaseId}`} failed: ${error.message}`)
|
||||
await api.post('/api/v1/seed/inspect/failed', { ...ref, error: error.message }).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private connect() {
|
||||
return XoverClient.connect({
|
||||
host: nntpConfig.host,
|
||||
port: nntpConfig.port,
|
||||
secure: nntpConfig.secure,
|
||||
servername: nntpConfig.tlsServername,
|
||||
user: nntpConfig.user,
|
||||
password: nntpConfig.password?.release(),
|
||||
timeoutMs: nntpConfig.commandTimeoutMs,
|
||||
compress: true,
|
||||
})
|
||||
}
|
||||
|
||||
/** The next range, or null when there is none (or the API can't be reached; logged). */
|
||||
private async lease(api: SeedApi, log: (message: string) => void): Promise<Lease | null> {
|
||||
try {
|
||||
return ((await api.post('/api/v1/seed/lease')) as { range: Lease } | null)?.range ?? null
|
||||
} catch (error: any) {
|
||||
if (error instanceof Unauthorized) throw error
|
||||
log(`Lease failed: ${error.message}`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** Works one leased range; false when the NNTP connection should be replaced. */
|
||||
private async workRange(
|
||||
api: SeedApi,
|
||||
client: OverviewSource,
|
||||
lease: Lease,
|
||||
log: (message: string) => void
|
||||
): Promise<boolean> {
|
||||
log(`${lease.group} ${lease.start}–${lease.end}`)
|
||||
const started = Date.now()
|
||||
let headers = 0
|
||||
let chain: Promise<void> = Promise.resolve()
|
||||
// Where the range resumes after what was stored: a batch split in two reports it with its first half.
|
||||
let reported = lease.start
|
||||
const posting = new Set<Promise<void>>()
|
||||
// Throws once a post has failed, so the range stops fetching.
|
||||
const failedSoFar = () => Promise.race([chain, Promise.resolve()])
|
||||
try {
|
||||
await runRange({
|
||||
client,
|
||||
group: lease.group,
|
||||
start: lease.start,
|
||||
end: lease.end,
|
||||
chunk: this.chunk,
|
||||
submitEvery: this.submitEvery,
|
||||
isDeferred: (header: OverviewHeader) =>
|
||||
indexerConfig.deferBodyMessageIds.some((re) => re.test(header['message-id'] ?? '')),
|
||||
// Batches are posted in order, one after the other, while fetching goes on; with two
|
||||
// waiting, fetching waits too. A failed post stops the chain, so no batch is stored
|
||||
// past a missing one (the range would resume beyond it).
|
||||
sink: async (batch: SeedBatch) => {
|
||||
if (this.stopping) throw new Stopping()
|
||||
while (posting.size >= 2) await Promise.race(posting)
|
||||
const post = chain.then(async () => {
|
||||
await this.postBatch(api, lease.id, batch, reported)
|
||||
reported = batch.next
|
||||
headers += batch.headers
|
||||
})
|
||||
chain = post
|
||||
posting.add(post)
|
||||
post.catch(() => {}).finally(() => posting.delete(post))
|
||||
await failedSoFar()
|
||||
},
|
||||
})
|
||||
await chain
|
||||
await api.post(`/api/v1/seed/ranges/${lease.id}/done`)
|
||||
const seconds = (Date.now() - started) / 1000
|
||||
log(
|
||||
`done: ${headers.toLocaleString('en')} headers in ${seconds.toFixed(0)} s (${Math.round(headers / seconds).toLocaleString('en')}/s)`
|
||||
)
|
||||
return true
|
||||
} catch (error: any) {
|
||||
if (error instanceof Unauthorized) throw error
|
||||
// An XOVER may still be in flight: the connection is replaced either way.
|
||||
if (error instanceof LeaseLost) {
|
||||
log(`Range ${lease.id} passed to another node; dropping it.`)
|
||||
return false
|
||||
}
|
||||
const reason = error instanceof Stopping ? 'node stopping' : error.message
|
||||
if (!(error instanceof Stopping)) log(`Range ${lease.id} failed: ${reason}`)
|
||||
// Let posts in flight finish (or fail) first, so the range resumes after what was stored.
|
||||
await chain.catch(() => {})
|
||||
await api.post(`/api/v1/seed/ranges/${lease.id}/release`, { error: reason }).catch(() => {})
|
||||
if (!this.stopping) await this.pause(30_000)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Posts a batch; one the seed API finds too large (413: dense recent ranges) is
|
||||
* posted in two halves, at most three splits deep. Only the second half moves
|
||||
* the range's resume point, so a failure in between loses nothing.
|
||||
*/
|
||||
private async postBatch(api: SeedApi, rangeId: number, batch: SeedBatch, reported: number, depth = 0): Promise<void> {
|
||||
try {
|
||||
await api.post(`/api/v1/seed/ranges/${rangeId}/batches`, batch)
|
||||
} catch (error: any) {
|
||||
const parts = batch.files.length + batch.fragments.length
|
||||
if (!/: 413 /.test(String(error.message)) || depth >= 3 || parts < 2) throw error
|
||||
const files = Math.ceil(batch.files.length / 2)
|
||||
const fragments = Math.ceil(batch.fragments.length / 2)
|
||||
const first: SeedBatch = { ...batch, next: reported, headers: 0, obfuscated: 0, deferred: 0, files: batch.files.slice(0, files), fragments: batch.fragments.slice(0, fragments) }
|
||||
const second: SeedBatch = { ...batch, files: batch.files.slice(files), fragments: batch.fragments.slice(fragments) }
|
||||
await this.postBatch(api, rangeId, first, reported, depth + 1)
|
||||
await this.postBatch(api, rangeId, second, reported, depth + 1)
|
||||
}
|
||||
}
|
||||
|
||||
/** Sleeps, but wakes up early when the node is stopping. */
|
||||
private async pause(ms: number) {
|
||||
for (let waited = 0; waited < ms && !this.stopping; waited += 1000) await sleep(1000)
|
||||
}
|
||||
}
|
||||
|
||||
class Unauthorized extends Error {}
|
||||
|
||||
/** The seed API client: JSON over HTTPS, gzip request bodies, retries for network errors and 5xx. */
|
||||
class SeedApi {
|
||||
constructor(
|
||||
private base: string,
|
||||
private token: string,
|
||||
private warn: (message: string) => void
|
||||
) {}
|
||||
|
||||
async post(path: string, body?: unknown): Promise<unknown> {
|
||||
const payload = body === undefined ? undefined : gzipSync(JSON.stringify(body))
|
||||
const result = await this.request('POST', path, payload)
|
||||
if (result === null) return null
|
||||
const text = result.toString('utf8')
|
||||
return text ? JSON.parse(text) : null
|
||||
}
|
||||
|
||||
/** A binary GET (a relay chunk, an article). */
|
||||
async get(path: string, attempts = 14): Promise<Buffer> {
|
||||
return (await this.request('GET', path, undefined, attempts)) ?? Buffer.alloc(0)
|
||||
}
|
||||
|
||||
private async request(method: string, path: string, payload?: Buffer, attempts = 14): Promise<Buffer | null> {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
let status = 0
|
||||
let data = Buffer.alloc(0)
|
||||
try {
|
||||
const response = await fetch(this.base + path, {
|
||||
method,
|
||||
headers: {
|
||||
authorization: `Bearer ${this.token}`,
|
||||
...(payload ? { 'content-type': 'application/json', 'content-encoding': 'gzip' } : {}),
|
||||
},
|
||||
body: payload,
|
||||
signal: AbortSignal.timeout(10 * 60 * 1000),
|
||||
})
|
||||
status = response.status
|
||||
data = Buffer.from(await response.arrayBuffer())
|
||||
} catch (error: any) {
|
||||
data = Buffer.from(error.message)
|
||||
}
|
||||
const text = () => data.toString('utf8').slice(0, 300)
|
||||
if (status === 204) return null
|
||||
if (status >= 200 && status < 300) return data
|
||||
if (status === 401) throw new Unauthorized('The seed API rejected this node token')
|
||||
if (status === 409) throw new LeaseLost(text())
|
||||
// 404 comes from the proxy while the seed API restarts (a deploy): this API has no 404s of its own.
|
||||
if (status >= 400 && status < 500 && status !== 404)
|
||||
throw new Error(`Seed API ${path}: ${status} ${text()}`)
|
||||
// Network errors, 404 and 5xx: storing a batch twice is harmless, so try again.
|
||||
// ~20 minutes in all, longer than a lease lasts: a restart or maintenance of the API is waited out.
|
||||
if (attempt >= attempts) throw new Error(`Seed API ${path}: ${status || 'no response'} ${text()}`)
|
||||
const wait = Math.min(120_000, 2000 * 2 ** (attempt - 1))
|
||||
this.warn(`Seed API ${path}: ${status || text()}; retrying in ${wait / 1000} s`)
|
||||
await sleep(wait)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** A range's overviews through the seed API's relay (CONCEPT.md D38): no news server account on the node. */
|
||||
class RelaySource implements OverviewSource {
|
||||
constructor(
|
||||
private api: SeedApi,
|
||||
private rangeId: number
|
||||
) {}
|
||||
|
||||
async group() {}
|
||||
|
||||
async xover(from: number, to: number): Promise<Overview[]> {
|
||||
const raw = await this.api.get(
|
||||
`/api/v1/seed/ranges/${this.rangeId}/chunk?from=${from}&to=${to}`
|
||||
)
|
||||
return raw.length > 0 ? parseOverviews(unzipSync(raw).toString('utf8')) : []
|
||||
}
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import NntpService from '#services/NntpService'
|
||||
import SeedOrchestrator from '#services/SeedOrchestrator'
|
||||
|
||||
/**
|
||||
* Plans seed ranges (CONCEPT.md D38) for groups: every article from below
|
||||
* what live indexing and the backfill cover down to `--days` back (or the
|
||||
* server's oldest article) that no earlier range covers. Running it again
|
||||
* plans only what's missing. A group's backfill stops where its plan
|
||||
* starts. Ranges are interleaved across groups, newest first.
|
||||
*
|
||||
* node ace seed:plan alt.binaries.ath alt.binaries.mom --days 365
|
||||
*/
|
||||
export default class SeedPlan extends BaseCommand {
|
||||
public static commandName = 'seed:plan'
|
||||
public static description = 'Plans article ranges of groups for seeding nodes to lease.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@args.spread({ description: 'Groups' })
|
||||
declare groups: string[]
|
||||
|
||||
@flags.number({ description: 'How many days back (default: everything the server has)' })
|
||||
declare days?: number
|
||||
|
||||
@flags.number({ description: 'Articles per range', default: 2_000_000 })
|
||||
declare size: number
|
||||
|
||||
@flags.number({ description: 'Leased before ranges of a lower priority (default 0)', default: 0 })
|
||||
declare priority: number
|
||||
|
||||
public async run() {
|
||||
try {
|
||||
const summary = await SeedOrchestrator.plan(this.groups, { days: this.days, size: this.size, priority: this.priority })
|
||||
for (const s of summary) {
|
||||
this.logger.info(
|
||||
`${s.group}: ${s.ranges.toLocaleString('en')} new ranges in ${s.gaps} gap(s) not yet planned, articles ${s.from.toLocaleString('en')}–${s.to.toLocaleString('en')}.`
|
||||
)
|
||||
}
|
||||
} finally {
|
||||
await NntpService.shutdown()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,126 +0,0 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import nntpConfig from '#config/nntp'
|
||||
import SeedService from '#services/SeedService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import { isDeferred } from '#services/DeferredService'
|
||||
import { XoverClient } from '#lib/nntp/xover_client'
|
||||
import { runRange } from '#lib/seed/runner'
|
||||
import type { SeedBatch } from '#lib/seed/wire'
|
||||
|
||||
/**
|
||||
* Edge aggregation (CONCEPT.md D37) on this machine: indexes a range of a
|
||||
* group's history over one NNTP connection (gzip-compressed overviews when
|
||||
* the server has them), grouping headers into files in memory and storing
|
||||
* one row per file directly. seed:node does the same for a remote node,
|
||||
* through the seed API. Header-only, like the backfill.
|
||||
*
|
||||
* node ace seed:range alt.binaries.ath --start 46000000000 --count 2000000
|
||||
* node ace seed:range alt.binaries.ath --start 46000000000 --count 2000000 --dry-run
|
||||
*/
|
||||
export default class SeedRange extends BaseCommand {
|
||||
public static commandName = 'seed:range'
|
||||
public static description = "Indexes a range of a group's headers by aggregating them into files on this node."
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@args.string({ description: 'Group' })
|
||||
declare group: string
|
||||
|
||||
@flags.string({ description: 'First article number' })
|
||||
declare start: string
|
||||
|
||||
@flags.number({ description: 'Articles to index', default: 1_000_000 })
|
||||
declare count: number
|
||||
|
||||
@flags.number({ description: 'Articles per XOVER', default: 20_000 })
|
||||
declare chunk: number
|
||||
|
||||
@flags.number({ description: 'Batches without a new part before an open file is given up as fragments', default: 5 })
|
||||
declare idle: number
|
||||
|
||||
@flags.boolean({ description: 'Fetch and aggregate only; store nothing' })
|
||||
declare dryRun: boolean
|
||||
|
||||
@flags.boolean({ description: 'Ask the server for gzip-compressed overviews', default: true })
|
||||
declare compress: boolean
|
||||
|
||||
public async run() {
|
||||
const first = Number(this.start)
|
||||
if (!Number.isSafeInteger(first) || first < 1) throw new Error(`--start must be an article number, got ${this.start}`)
|
||||
const last = first + this.count - 1
|
||||
const stats = { headers: 0, obfuscated: 0, deferred: 0, complete: 0, files: 0, existing: 0, dropped: 0, fragments: 0, inspections: 0 }
|
||||
let storeMs = 0
|
||||
const cpuStart = process.cpuUsage()
|
||||
const started = Date.now()
|
||||
|
||||
const client = await XoverClient.connect({
|
||||
host: nntpConfig.host,
|
||||
port: nntpConfig.port,
|
||||
secure: nntpConfig.secure,
|
||||
servername: nntpConfig.tlsServername,
|
||||
user: nntpConfig.user,
|
||||
password: nntpConfig.password?.release(),
|
||||
timeoutMs: nntpConfig.commandTimeoutMs,
|
||||
compress: this.compress,
|
||||
})
|
||||
this.logger.info(`Connected${client.compressed ? ', overviews gzip-compressed' : ''}.`)
|
||||
try {
|
||||
await runRange({
|
||||
client,
|
||||
group: this.group,
|
||||
start: first,
|
||||
end: last,
|
||||
chunk: this.chunk,
|
||||
idle: this.idle,
|
||||
submitEvery: 1,
|
||||
isDeferred,
|
||||
sink: async (batch: SeedBatch) => {
|
||||
stats.headers += batch.headers
|
||||
stats.obfuscated += batch.obfuscated
|
||||
stats.deferred += batch.deferred
|
||||
stats.complete += batch.files.length
|
||||
stats.fragments += batch.fragments.length
|
||||
if (!this.dryRun) {
|
||||
const t = Date.now()
|
||||
const stored = await SeedService.store(batch.files)
|
||||
await SeedService.storeFragments(batch.fragments, { start: first, end: last })
|
||||
storeMs += Date.now() - t
|
||||
stats.files += stored.files
|
||||
stats.existing += stored.existing
|
||||
stats.dropped += stored.dropped
|
||||
stats.inspections += stored.inspections
|
||||
}
|
||||
this.progress(stats, started)
|
||||
},
|
||||
})
|
||||
} finally {
|
||||
client.close()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
|
||||
const seconds = (Date.now() - started) / 1000
|
||||
const cpu = process.cpuUsage(cpuStart)
|
||||
const n = (x: number) => x.toLocaleString('en')
|
||||
this.logger.info(
|
||||
`${this.group} ${first}-${last}: ${n(stats.headers)} headers in ${seconds.toFixed(0)} s (${n(Math.round(stats.headers / seconds))}/s). ` +
|
||||
`${n(stats.obfuscated)} obfuscated, ${n(stats.deferred)} deferred. ` +
|
||||
`${n(stats.complete)} complete files (${n(stats.files)} new, ${n(stats.existing)} known, ${n(stats.dropped)} garbage), ` +
|
||||
`${n(stats.fragments)} fragment segments, ${n(stats.inspections)} queued for inspection.`
|
||||
)
|
||||
this.logger.info(`Store ${(storeMs / 1000).toFixed(0)} s; node CPU ${((cpu.user + cpu.system) / 1e6).toFixed(0)} s.`)
|
||||
}
|
||||
|
||||
private lastReport = 0
|
||||
private progress(stats: Record<string, number>, started: number) {
|
||||
if (Date.now() - this.lastReport < 30_000) return
|
||||
this.lastReport = Date.now()
|
||||
const seconds = (Date.now() - started) / 1000
|
||||
this.logger.info(
|
||||
`${stats.headers.toLocaleString('en')} headers (${Math.round(stats.headers / seconds).toLocaleString('en')}/s), ` +
|
||||
`${stats.complete.toLocaleString('en')} files, ${stats.fragments.toLocaleString('en')} fragment segments.`
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import SeedOrchestrator from '#services/SeedOrchestrator'
|
||||
|
||||
/** Seeding progress per range status and per node (CONCEPT.md D38). */
|
||||
export default class SeedStatus extends BaseCommand {
|
||||
public static commandName = 'seed:status'
|
||||
public static description = 'Shows seeding progress per range status and per node.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
public async run() {
|
||||
const status = await SeedOrchestrator.status()
|
||||
const n = (x: unknown) => Number(x ?? 0).toLocaleString('en')
|
||||
for (const r of status.ranges) {
|
||||
this.logger.info(`${r.status}: ${n(r.ranges)} ranges, ${n(r.reached)} of ${n(r.articles)} articles reached, ${n(r.headers)} headers, ${n(r.files)} files`)
|
||||
}
|
||||
for (const node of status.nodes) {
|
||||
this.logger.info(`node ${node.name}${node.disabled ? ' (disabled)' : ''}: ${node.leases} leases, ${n(node.headers)} headers, last seen ${node.last_seen_at ?? 'never'}`)
|
||||
}
|
||||
this.logger.info(`Headers in ranges finished in the last hour: ${n(status.headersDoneLastHour)}`)
|
||||
if (status.paused) this.logger.warning(`No new leases: ${status.paused} (BACKFILL_MAX_NVME_GB / BACKFILL_MAX_HDD_GB).`)
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import SeedOrchestrator from '#services/SeedOrchestrator'
|
||||
|
||||
/**
|
||||
* Creates a seeding node, or gives it a new token (the old one stops
|
||||
* working), and prints the token once (CONCEPT.md D38).
|
||||
*
|
||||
* node ace seed:token hetzner-1
|
||||
* node ace seed:token hetzner-1 --disable
|
||||
*/
|
||||
export default class SeedToken extends BaseCommand {
|
||||
public static commandName = 'seed:token'
|
||||
public static description = 'Creates a seeding node or replaces its token; prints the token once.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@args.string({ description: 'Node name' })
|
||||
declare name: string
|
||||
|
||||
@flags.boolean({ description: 'Disable the node instead (its leases pass to others after they expire)' })
|
||||
declare disable: boolean
|
||||
|
||||
public async run() {
|
||||
if (this.disable) {
|
||||
const n = await db.from('seed_nodes').where('name', this.name).update({ disabled: true })
|
||||
this.logger.info(n ? `${this.name}: disabled.` : `${this.name}: no such node.`)
|
||||
return
|
||||
}
|
||||
const token = await SeedOrchestrator.issueToken(this.name)
|
||||
this.logger.info(`${this.name}: token (shown once, keep it on the node only):`)
|
||||
console.log(token)
|
||||
}
|
||||
}
|
||||
@@ -1,25 +0,0 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import SpotService from '#services/SpotService'
|
||||
|
||||
/** Passwords from the titles and descriptions of spots read so far, onto their releases. */
|
||||
export default class SpotPasswords extends BaseCommand {
|
||||
public static commandName = 'spot:passwords'
|
||||
public static description = "Takes archive passwords from spots' titles and descriptions onto their releases."
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
public async run() {
|
||||
const spots = await db
|
||||
.from('spots')
|
||||
.whereNotNull('release_id')
|
||||
.where((q) => q.whereNotNull('description').orWhere('title', 'like', '%{{%}}%'))
|
||||
.select('message_id')
|
||||
let found = 0
|
||||
for (const spot of spots) if (await SpotService.applyPassword(spot.message_id)) found++
|
||||
this.logger.info(`${found} of ${spots.length} spots name a password in their title or description.`)
|
||||
}
|
||||
}
|
||||
@@ -1,55 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import env from '#start/env'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
import NntpService from '#services/NntpService'
|
||||
import SpotService from '#services/SpotService'
|
||||
|
||||
const BATCH = 500
|
||||
|
||||
/**
|
||||
* Reads the article and NZB of listed spots that were listed from their
|
||||
* overview only (LOG.md 2026-10-08 (10)): description, the NZB's password or
|
||||
* one in the spot's text, placeholder NZBs. Newest first; can be stopped and
|
||||
* started again (worker:spots does the same when idle, a batch at a time).
|
||||
* Connections: NNTP_POOL_SIZE of this process, --concurrency at a time.
|
||||
*/
|
||||
export default class SpotPostprocess extends BaseCommand {
|
||||
public static commandName = 'spot:postprocess'
|
||||
public static description = "Reads listed spots' articles and NZBs for their passwords and details."
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.number({ description: 'Spots read at a time (default: NNTP_POOL_SIZE)' })
|
||||
declare concurrency?: number
|
||||
|
||||
@flags.number({ description: 'Stop after this many spots' })
|
||||
declare limit?: number
|
||||
|
||||
public async run() {
|
||||
const concurrency = this.concurrency ?? env.get('NNTP_POOL_SIZE', 4)
|
||||
const [{ count }] = await db.from('spots').where('status', 'listed').whereNull('nzb_read_at').count('* as count')
|
||||
this.logger.info(`${count} listed spots to read, ${concurrency} at a time.`)
|
||||
const totals: Record<string, number> = {}
|
||||
const started = Date.now()
|
||||
let done = 0
|
||||
try {
|
||||
while (!Shutdown.requested && (!this.limit || done < this.limit)) {
|
||||
const counts = await SpotService.readListings(Math.min(BATCH, this.limit ? this.limit - done : BATCH), concurrency)
|
||||
const read = Object.values(counts).reduce((sum, n) => sum + n, 0)
|
||||
if (read === 0) break
|
||||
for (const [k, v] of Object.entries(counts)) totals[k] = (totals[k] ?? 0) + v
|
||||
done += read
|
||||
const rate = done / ((Date.now() - started) / 1000)
|
||||
this.logger.info(`${done} read (${rate.toFixed(1)}/s): ${Object.entries(totals).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
|
||||
// Usenet unreachable: everything was given back; don't spin on it.
|
||||
if (counts.retry === read) await Shutdown.sleep(30_000)
|
||||
}
|
||||
} finally {
|
||||
await NntpService.shutdown()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,51 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import NntpService from '#services/NntpService'
|
||||
import SpotService from '#services/SpotService'
|
||||
|
||||
/**
|
||||
* Imported spots whose files sit in a release that isn't the spot's own
|
||||
* (another source, or another name): their NZB is read again, which moves
|
||||
* the files into the spot's release under its title (CONCEPT.md D29).
|
||||
*/
|
||||
export default class SpotReimport extends BaseCommand {
|
||||
public static commandName = 'spot:reimport'
|
||||
public static description = "Moves imported spots' files into their own release, named after the spot."
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only count them' })
|
||||
declare dryRun: boolean
|
||||
|
||||
public async run() {
|
||||
const spots = await db
|
||||
.from('spots as s')
|
||||
.join('releases as r', 'r.id', 's.release_id')
|
||||
.where('s.status', 'imported')
|
||||
.where((q) => q.whereNot('r.source', 'spot').orWhereRaw('r.name <> s.title'))
|
||||
.orderBy('s.posted_at', 'desc')
|
||||
.select('s.*', 'r.name as release_name')
|
||||
this.logger.info(`${spots.length} imported spots are in a release that isn't their own.`)
|
||||
if (this.dryRun) return
|
||||
let moved = 0
|
||||
let failed = 0
|
||||
try {
|
||||
for (const spot of spots) {
|
||||
try {
|
||||
const releaseId = await SpotService.reimport(spot)
|
||||
if (releaseId) moved++
|
||||
else failed++
|
||||
} catch (error: any) {
|
||||
failed++
|
||||
this.logger.warning(`${spot.title}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await NntpService.shutdown()
|
||||
}
|
||||
this.logger.info(`${moved} spots re-imported, ${failed} without a usable NZB (dummy, gone) or failed.`)
|
||||
}
|
||||
}
|
||||
@@ -1,97 +0,0 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { Worker } from 'bullmq'
|
||||
import fs from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import queueConfig from '#config/queue'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
import QueueService from '#services/QueueService'
|
||||
import env from '#start/env'
|
||||
import NntpService from '#services/NntpService'
|
||||
import IngestService, { segmentFromYenc } from '#services/IngestService'
|
||||
import { parseYencMeta, YencError } from '#lib/yenc/index'
|
||||
import { firstChunkMd5 } from '#services/ContentService'
|
||||
|
||||
export default class BodyWorker extends BaseCommand {
|
||||
public static commandName = 'worker:body'
|
||||
public static description = 'Starts a worker to process article bodies for yEnc metadata.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
public async run() {
|
||||
this.logger.info('Starting body worker...')
|
||||
|
||||
const pool = NntpService
|
||||
|
||||
const worker = new Worker('body-queue', async (job) => {
|
||||
const { header, group, articleNumber } = job.data
|
||||
const messageId = header['message-id']
|
||||
this.logger.debug(`Processing header with unparsable subject: ${header.subject}`)
|
||||
|
||||
let conn
|
||||
let failed = false
|
||||
try {
|
||||
conn = await pool.acquire()
|
||||
const bodyBuffer = await pool.fetchBody(conn, messageId)
|
||||
|
||||
let meta
|
||||
try {
|
||||
meta = parseYencMeta(bodyBuffer)
|
||||
} catch (parseError: any) {
|
||||
if (parseError instanceof YencError && bodyBuffer.indexOf('=ybegin ') === -1) {
|
||||
this.logger.debug(`Not a yEnc post, skipping: ${messageId}`)
|
||||
return
|
||||
}
|
||||
this.logger.error(`Failed to parse yEnc data for message ID ${messageId}. Dumping buffer.`)
|
||||
const debugDir = path.join(this.app.appRoot.pathname, 'debug')
|
||||
await fs.mkdir(debugDir, { recursive: true })
|
||||
const timestamp = new Date().toISOString().replace(/:/g, '-')
|
||||
const dumpFile = path.join(debugDir, `body-error-${timestamp}-${messageId.replace(/[<>]/g, '')}.bin`)
|
||||
await fs.writeFile(dumpFile, bodyBuffer)
|
||||
this.logger.error(`Problematic body buffer saved to: ${dumpFile}`)
|
||||
throw parseError
|
||||
}
|
||||
|
||||
pool.release(conn)
|
||||
conn = undefined
|
||||
|
||||
const segment = segmentFromYenc(header, group, meta, articleNumber)
|
||||
if (segment?.number === 1) segment.md5_16k = firstChunkMd5(bodyBuffer, meta.begin.size)
|
||||
if (!segment) {
|
||||
this.logger.warning(`Inconsistent yEnc part numbering in ${messageId}, skipping.`)
|
||||
return
|
||||
}
|
||||
const completed = await IngestService.ingest([segment])
|
||||
this.logger.debug(
|
||||
`Stored part ${segment.number}/${segment.totalParts} of "${segment.name}"` +
|
||||
(completed.length > 0 ? ' (file complete)' : '')
|
||||
)
|
||||
} catch (error: any) {
|
||||
failed = true
|
||||
this.logger.error(`Error in body worker for message ID ${messageId}: ${error.message}`)
|
||||
throw error
|
||||
} finally {
|
||||
if (conn) {
|
||||
pool.release(conn, failed)
|
||||
}
|
||||
}
|
||||
}, {
|
||||
connection: queueConfig.connection,
|
||||
// Each job holds one NNTP connection while downloading, so keep this <= the pool size.
|
||||
concurrency: env.get('BODY_CONCURRENCY', 1),
|
||||
})
|
||||
|
||||
worker.on('failed', (job, err) => {
|
||||
this.logger.error(`Body job ${job?.id} failed: ${err.message}`)
|
||||
})
|
||||
|
||||
this.logger.info('Body worker started and listening for jobs.')
|
||||
await Shutdown.wait()
|
||||
this.logger.info('Shutting down: finishing active jobs...')
|
||||
await worker.close()
|
||||
await NntpService.shutdown()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { Worker } from 'bullmq'
|
||||
import queueConfig from '#config/queue'
|
||||
import env from '#start/env'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NntpService from '#services/NntpService'
|
||||
import ContentService, { describeContent } from '#services/ContentService'
|
||||
import ArchiveWalkService from '#services/ArchiveWalkService'
|
||||
import File from '#models/file'
|
||||
|
||||
export default class CollectionWorker extends BaseCommand {
|
||||
public static commandName = 'worker:collection'
|
||||
public static description = 'Starts a worker to process file collections (e.g., RAR archives).'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
public async run() {
|
||||
this.logger.info('Starting collection worker...')
|
||||
|
||||
const worker = new Worker('collection-queue', async (job) => {
|
||||
const { fileId } = job.data
|
||||
this.logger.debug(`Processing file ID ${fileId} for collection.`)
|
||||
|
||||
const file = await File.find(fileId)
|
||||
if (!file) {
|
||||
this.logger.error(`File with ID ${fileId} not found in the database.`)
|
||||
return
|
||||
}
|
||||
|
||||
try {
|
||||
const content = await ContentService.inspect(fileId)
|
||||
for (const line of describeContent(file.filename, content)) this.logger.info(line)
|
||||
} catch (error: any) {
|
||||
this.logger.warning(`Content inspection of "${file.filename}" failed: ${error.message}`)
|
||||
}
|
||||
}, {
|
||||
connection: queueConfig.connection,
|
||||
// Each job holds one NNTP connection while downloading; keep <= the pool size.
|
||||
concurrency: env.get('COLLECTION_CONCURRENCY', 1),
|
||||
})
|
||||
|
||||
worker.on('failed', (job, err) => {
|
||||
this.logger.error(`Collection job ${job?.id} failed: ${err.message}`)
|
||||
})
|
||||
|
||||
// Complete archive listings (D40): background work beside inspection, on the same NNTP pool.
|
||||
const walkConcurrency = env.get('ARCHIVE_WALK_CONCURRENCY', 1)
|
||||
const walker = walkConcurrency > 0
|
||||
? new Worker('archive-walk-queue', async (job) => {
|
||||
const o = await ArchiveWalkService.walk(Number(job.data.releaseId))
|
||||
if (o.skipped) return o
|
||||
this.logger.info(
|
||||
`Archive walk of release ${o.releaseId}: ${o.before} → ${o.entries} entries from ${o.segments} segments` +
|
||||
(o.complete ? '' : ` (stopped: ${o.stopped})`) +
|
||||
(o.renamed ? `, renamed to "${o.renamed}"` : '')
|
||||
)
|
||||
return o
|
||||
}, { connection: queueConfig.connection, concurrency: walkConcurrency })
|
||||
: null
|
||||
walker?.on('failed', (job, err) => {
|
||||
this.logger.error(`Archive walk job ${job?.id} failed: ${err.message}`)
|
||||
})
|
||||
|
||||
this.logger.info('Collection worker started and listening for jobs.')
|
||||
await Shutdown.wait()
|
||||
this.logger.info('Shutting down: finishing active jobs...')
|
||||
await Promise.all([worker.close(), walker?.close()])
|
||||
await NntpService.shutdown()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
@@ -1,63 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import env from '#start/env'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import DeferredService from '#services/DeferredService'
|
||||
|
||||
const IDLE_MS = 60_000
|
||||
|
||||
/**
|
||||
* Works through `deferred_files` (CONCEPT.md D28): one body per file, and
|
||||
* small files completely. Not started by default (compose profile
|
||||
* `deferred`); run it when there is bandwidth to spare.
|
||||
*/
|
||||
export default class DeferredWorker extends BaseCommand {
|
||||
public static commandName = 'worker:deferred'
|
||||
public static description = 'Samples deferred files (e.g. ngPost uploads): one article body per file.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.number({ description: 'Stop after this many files (default: keep running)' })
|
||||
declare limit?: number
|
||||
|
||||
public async run() {
|
||||
const concurrency = env.get('DEFERRED_CONCURRENCY', 4)
|
||||
this.logger.info(`Deferred worker started (concurrency ${concurrency}).`)
|
||||
const counts: Record<string, number> = {}
|
||||
let done = 0
|
||||
try {
|
||||
while (!Shutdown.requested && (!this.limit || done < this.limit)) {
|
||||
try {
|
||||
const batch = await DeferredService.claim(Math.min(concurrency * 4, this.limit ? this.limit - done : Infinity))
|
||||
if (batch.length === 0) {
|
||||
if (this.limit) break
|
||||
await Shutdown.sleep(IDLE_MS)
|
||||
continue
|
||||
}
|
||||
let next = 0
|
||||
await Promise.all(
|
||||
Array.from({ length: concurrency }, async () => {
|
||||
while (next < batch.length) {
|
||||
const status = await DeferredService.process(batch[next++])
|
||||
counts[status] = (counts[status] ?? 0) + 1
|
||||
}
|
||||
})
|
||||
)
|
||||
done += batch.length
|
||||
this.logger.info(`${done} deferred files processed: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
|
||||
} catch (err: any) {
|
||||
// E.g. Postgres restarting: wait and claim again, instead of ending the loop (LOG.md 2026-10-07 (3)).
|
||||
this.logger.error(`Round failed, retrying in 30 s: ${err.message}`)
|
||||
await Shutdown.sleep(30_000)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await NntpService.shutdown()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,118 +0,0 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { Worker } from 'bullmq'
|
||||
import queueConfig from '#config/queue'
|
||||
import env from '#start/env'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NntpService from '#services/NntpService'
|
||||
import FetchRangeService from '#services/FetchRangeService'
|
||||
import IngestService, { segmentFromHeader, type IngestSegment } from '#services/IngestService'
|
||||
import DeferredService, { bodyJob, isDeferred, type DeferredArticle } from '#services/DeferredService'
|
||||
import SpotService, { isSpotGroup } from '#services/SpotService'
|
||||
import StagingService from '#services/StagingService'
|
||||
|
||||
export default class FetchWorker extends BaseCommand {
|
||||
public static commandName = 'worker:fetch'
|
||||
public static description = 'Starts a worker that fetches headers (XOVER) and stores them as binaries/parts.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
public async run() {
|
||||
this.logger.info('Starting fetch worker...')
|
||||
|
||||
const pool = NntpService
|
||||
const bodyQueue = QueueService.bodyQueue
|
||||
|
||||
const worker = new Worker('nntp-fetch-queue', async (job) => {
|
||||
const { groupName, startId, endId } = job.data
|
||||
this.logger.info(`Processing fetch job for ${groupName}, articles ${startId}-${endId}`)
|
||||
|
||||
let overviews: [number, Record<string, string>][]
|
||||
let conn
|
||||
let failed = false
|
||||
try {
|
||||
conn = await pool.acquire()
|
||||
await pool.timed(conn.group(groupName), `GROUP ${groupName}`)
|
||||
const reply = await pool.timed<any>(conn.xover(startId, endId), `XOVER ${groupName} ${startId}-${endId}`)
|
||||
// An empty reply parses as one line without fields ([null, {}]).
|
||||
overviews = reply.overviews.filter(([number, header]: [number | null, Record<string, string>]) => number !== null && header?.['message-id'])
|
||||
} catch (error: any) {
|
||||
failed = true
|
||||
this.logger.error(`Error fetching headers for ${groupName}: ${error.message}`)
|
||||
throw error
|
||||
} finally {
|
||||
if (conn) {
|
||||
pool.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
if (isSpotGroup(groupName)) {
|
||||
const spots = await SpotService.recordHeaders(groupName, overviews)
|
||||
if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
|
||||
this.logger.info(`${groupName} ${startId}-${endId}: ${overviews.length} headers, ${spots} spots queued.`)
|
||||
return
|
||||
}
|
||||
|
||||
const group = await db.from('groups').where('name', groupName).first()
|
||||
// Backfill is header-only: old bodies would be far more than the link carries (CONCEPT.md D32).
|
||||
const fetchBodies = Boolean(group?.fetch_bodies) && !job.data.backfill
|
||||
|
||||
const segments: IngestSegment[] = []
|
||||
const obfuscated: { articleNumber: number; header: Record<string, string> }[] = []
|
||||
const deferred: DeferredArticle[] = []
|
||||
let skipped = 0
|
||||
for (const [articleNumber, header] of overviews) {
|
||||
if (!header) continue
|
||||
const segment = segmentFromHeader(header, groupName, articleNumber)
|
||||
if (segment) segments.push(segment)
|
||||
// Recorded per file (or as article ranges) in every group, bodies or not; processed later if at all.
|
||||
else if (isDeferred(header)) deferred.push({ articleNumber, header })
|
||||
else if (!fetchBodies || indexerConfig.skipBodyMessageIds.some((re) => re.test(header['message-id'] ?? ''))) skipped++
|
||||
else obfuscated.push({ articleNumber, header })
|
||||
}
|
||||
|
||||
// Lone segments of new files wait in staging until their file proves real (D35).
|
||||
const { ingest, staged } = StagingService.enabled(groupName)
|
||||
? await StagingService.filter(segments, { counters: StagingService.countersEnabled(groupName) })
|
||||
: { ingest: segments, staged: 0 }
|
||||
const completed = await IngestService.ingest(ingest)
|
||||
const deferredFiles = await DeferredService.record(groupName, deferred)
|
||||
|
||||
if (obfuscated.length > 0) {
|
||||
await bodyQueue.addBulk(obfuscated.map(({ articleNumber, header }) => bodyJob(header, groupName, articleNumber)))
|
||||
}
|
||||
|
||||
if (job.data.rangeId) await FetchRangeService.markDone(job.data.rangeId)
|
||||
|
||||
this.logger.info(
|
||||
`${groupName} ${startId}-${endId}: ${overviews.length} headers, ${ingest.length} stored${staged ? `, ${staged} staged` : ''}, ` +
|
||||
`${obfuscated.length} sent to body-queue${skipped ? ` (${skipped} skipped)` : ''}` +
|
||||
`${deferred.length ? `, ${deferred.length} deferred (${deferredFiles} files)` : ''}, ${completed.length} files completed.`
|
||||
)
|
||||
}, {
|
||||
connection: queueConfig.connection,
|
||||
// XOVER is quick; most of a job is storing. Each concurrent job uses one DB connection (DB_POOL_MAX).
|
||||
concurrency: env.get('FETCH_CONCURRENCY', 5),
|
||||
})
|
||||
|
||||
worker.on('failed', async (job, err) => {
|
||||
this.logger.error(`Fetch job ${job?.id} failed for group ${job?.data.groupName}: ${err.message}`)
|
||||
// Only after the last attempt; the scheduler re-queues the range later.
|
||||
if (job?.data.rangeId && job.attemptsMade >= (job.opts.attempts ?? 1)) {
|
||||
await FetchRangeService.markFailed(job.data.rangeId, err.message).catch(() => {})
|
||||
}
|
||||
})
|
||||
|
||||
this.logger.info('Fetch worker started and listening for jobs.')
|
||||
await Shutdown.wait()
|
||||
this.logger.info('Shutting down: finishing active jobs...')
|
||||
await worker.close()
|
||||
await NntpService.shutdown()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import { Worker } from 'bullmq'
|
||||
import env from '#start/env'
|
||||
import BinaryFileService from '#services/BinaryFileService'
|
||||
import queueConfig from '#config/queue'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
import QueueService from '#services/QueueService'
|
||||
|
||||
export default class FileWorker extends BaseCommand {
|
||||
public static commandName = 'worker:file'
|
||||
public static description = 'Starts a worker that turns completed binaries into files.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
public async run() {
|
||||
this.logger.info('Starting file worker...')
|
||||
|
||||
const worker = new Worker('file-queue', async (job) => {
|
||||
// A batch of completed binaries ('process-files'), or one from before batching ('process-file').
|
||||
const ids: number[] = (job.data.binaryIds ?? [job.data.binaryId]).map(Number)
|
||||
const result = await BinaryFileService.store(ids)
|
||||
if (result.saved || result.existing) {
|
||||
this.logger.info(
|
||||
`Files: ${result.saved} saved, ${result.existing} existed already` +
|
||||
(result.skipped ? `, ${result.skipped} skipped` : '') +
|
||||
(result.inspections ? `, ${result.inspections} queued for inspection` : '') +
|
||||
` (${ids.length} binaries).`
|
||||
)
|
||||
}
|
||||
// The rest is stored; the job's retry picks these up (the others are linked now and skipped).
|
||||
if (result.incomplete.length > 0) {
|
||||
throw new Error(`${result.incomplete.length} of ${ids.length} binaries don't have all their parts yet.`)
|
||||
}
|
||||
}, {
|
||||
connection: queueConfig.connection,
|
||||
// One job at a time could not keep up with the backfill (130k waiting); jobs are per binary and
|
||||
// the file insert ignores duplicates, so they can run side by side.
|
||||
concurrency: env.get('FILE_CONCURRENCY', 8),
|
||||
})
|
||||
|
||||
worker.on('failed', (job, err) => {
|
||||
this.logger.error(`File job ${job?.id} failed: ${err.message}`)
|
||||
})
|
||||
|
||||
this.logger.info('File worker started and listening for jobs.')
|
||||
await Shutdown.wait()
|
||||
this.logger.info('Shutting down: finishing active jobs...')
|
||||
await worker.close()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
@@ -1,76 +0,0 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import env from '#start/env'
|
||||
import Shutdown from '#services/Shutdown'
|
||||
import NntpService from '#services/NntpService'
|
||||
import SpotService from '#services/SpotService'
|
||||
|
||||
const IDLE_MS = 30_000
|
||||
/** Spots claimed at a time; most of a history backfill is listed in bulk. */
|
||||
const BATCH = 500
|
||||
/** Imported spots whose details are read per idle round. */
|
||||
const DETAILS_BATCH = 50
|
||||
/** Listed spots whose article and NZB are read per idle round (passwords, dummies). */
|
||||
const LISTINGS_BATCH = 50
|
||||
|
||||
/**
|
||||
* Works through queued Spotnet spots (CONCEPT.md D29): verifies each one,
|
||||
* downloads its compressed NZB (a few KB to a few MB) and imports it. When
|
||||
* idle, reads what listed spots were listed without: details, passwords.
|
||||
*/
|
||||
export default class SpotsWorker extends BaseCommand {
|
||||
public static commandName = 'worker:spots'
|
||||
public static description = 'Reads Spotnet spots: verifies them and imports their NZBs.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.number({ description: 'Stop after this many spots (default: keep running)' })
|
||||
declare limit?: number
|
||||
|
||||
public async run() {
|
||||
const concurrency = env.get('SPOT_CONCURRENCY', 4)
|
||||
this.logger.info(`Spot worker started (concurrency ${concurrency}).`)
|
||||
const counts: Record<string, number> = {}
|
||||
let done = 0
|
||||
try {
|
||||
while (!Shutdown.requested && (!this.limit || done < this.limit)) {
|
||||
try {
|
||||
const batch = await SpotService.claim(Math.min(BATCH, this.limit ? this.limit - done : Infinity))
|
||||
if (batch.length === 0) {
|
||||
if (this.limit) break
|
||||
// Idle: read the details of imported spots that don't have them yet (newest first), then
|
||||
// the article and NZB of listed spots (their passwords; LOG.md 2026-10-08 (10)).
|
||||
if ((await SpotService.fillDetails(DETAILS_BATCH)) > 0) continue
|
||||
const listings = await SpotService.readListings(LISTINGS_BATCH, concurrency)
|
||||
const read = Object.values(listings).reduce((sum, n) => sum + n, 0)
|
||||
if (read === 0 || listings.retry === read) await Shutdown.sleep(IDLE_MS)
|
||||
continue
|
||||
}
|
||||
// Old, verified spots are listed in bulk; the rest need their header (and maybe NZB) read.
|
||||
const listable = batch.filter((spot) => SpotService.isListable(spot))
|
||||
const single = batch.filter((spot) => !SpotService.isListable(spot))
|
||||
counts.listed = (counts.listed ?? 0) + (await SpotService.list(listable))
|
||||
let next = 0
|
||||
await Promise.all(
|
||||
Array.from({ length: concurrency }, async () => {
|
||||
while (next < single.length) {
|
||||
const status = await SpotService.process(single[next++])
|
||||
counts[status] = (counts[status] ?? 0) + 1
|
||||
}
|
||||
})
|
||||
)
|
||||
done += batch.length
|
||||
this.logger.info(`${done} spots: ${Object.entries(counts).map(([k, v]) => `${v} ${k}`).join(', ')}.`)
|
||||
} catch (err: any) {
|
||||
// E.g. Postgres restarting: wait and claim again, instead of ending the loop (LOG.md 2026-10-07 (3)).
|
||||
this.logger.error(`Round failed, retrying in 30 s: ${err.message}`)
|
||||
await Shutdown.sleep(30_000)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await NntpService.shutdown()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -32,6 +32,10 @@ export const http = defineConfig({
|
||||
*/
|
||||
useAsyncLocalStorage: false,
|
||||
|
||||
router: {
|
||||
matcher: 'tree',
|
||||
},
|
||||
|
||||
/**
|
||||
* Redirect configuration controls the behavior of
|
||||
* response.redirect().back() and query string forwarding.
|
||||
|
||||
@@ -32,11 +32,6 @@ const bodyParserConfig = defineConfig({
|
||||
*/
|
||||
convertEmptyStringsToNull: true,
|
||||
|
||||
/**
|
||||
* Seeding nodes post batches of a few MB (CONCEPT.md D38); the limit applies after gzip.
|
||||
*/
|
||||
limit: '64mb',
|
||||
|
||||
/**
|
||||
* Content types handled by the JSON parser.
|
||||
*/
|
||||
|
||||
+14
-115
@@ -1,137 +1,36 @@
|
||||
import env from '#start/env'
|
||||
import app from '@adonisjs/core/services/app'
|
||||
import { defineConfig } from '@adonisjs/lucid'
|
||||
import env from '#start/env'
|
||||
|
||||
/**
|
||||
* The rebuild's own database (REBUILD.md, decision b), on the same server
|
||||
* as the current indexer's.
|
||||
*/
|
||||
const dbConfig = defineConfig({
|
||||
/**
|
||||
* Default connection used for all queries.
|
||||
*/
|
||||
connection: env.get('DB_CONNECTION', 'sqlite'),
|
||||
|
||||
connection: 'pg',
|
||||
connections: {
|
||||
/**
|
||||
* SQLite connection (default).
|
||||
*/
|
||||
sqlite: {
|
||||
client: 'better-sqlite3',
|
||||
|
||||
connection: {
|
||||
filename: app.tmpPath('db.sqlite3'),
|
||||
},
|
||||
|
||||
/**
|
||||
* Required by Knex for SQLite defaults.
|
||||
*/
|
||||
useNullAsDefault: true,
|
||||
|
||||
migrations: {
|
||||
/**
|
||||
* Sort migration files naturally by filename.
|
||||
*/
|
||||
naturalSort: true,
|
||||
|
||||
/**
|
||||
* Paths containing migration files.
|
||||
*/
|
||||
paths: ['database/migrations'],
|
||||
},
|
||||
|
||||
schemaGeneration: {
|
||||
/**
|
||||
* Enable schema generation from Lucid models.
|
||||
*/
|
||||
enabled: true,
|
||||
|
||||
/**
|
||||
* Custom schema rules file paths.
|
||||
*/
|
||||
rulesPaths: ['./database/schema_rules.js'],
|
||||
},
|
||||
},
|
||||
|
||||
/**
|
||||
* PostgreSQL connection.
|
||||
* Install package to switch: npm install pg
|
||||
*/
|
||||
pg: {
|
||||
client: 'pg',
|
||||
connection: {
|
||||
host: env.get('DB_HOST'),
|
||||
port: env.get('DB_PORT'),
|
||||
user: env.get('DB_USER'),
|
||||
password: env.get('DB_PASSWORD')?.release(),
|
||||
password: env.get('DB_PASSWORD'),
|
||||
database: env.get('DB_DATABASE'),
|
||||
// For idempotent bulk writers (worker:fetch): don't wait for the WAL
|
||||
// flush on every statement. A crash loses the last moments of writes,
|
||||
// which the retried job writes again.
|
||||
...(env.get('DB_SYNCHRONOUS_COMMIT') === 'off' ? { options: '-c synchronous_commit=off' } : {}),
|
||||
},
|
||||
// One connection per concurrent job; knex defaults to 10.
|
||||
// Local development may share a database, in its own schema (DB_SEARCH_PATH=rebuild).
|
||||
...(env.get('DB_SEARCH_PATH') ? { searchPath: env.get('DB_SEARCH_PATH')!.split(',') } : {}),
|
||||
pool: { min: 0, max: env.get('DB_POOL_MAX', 10) },
|
||||
migrations: {
|
||||
naturalSort: true,
|
||||
paths: ['database/migrations'],
|
||||
},
|
||||
debug: app.inDev,
|
||||
schemaGeneration: {
|
||||
enabled: true,
|
||||
rulesPaths: ['./database/schema_rules.js'],
|
||||
},
|
||||
debug: app.inDev && env.get('DB_DEBUG', false),
|
||||
},
|
||||
|
||||
/**
|
||||
* MySQL / MariaDB connection.
|
||||
* Install package to switch: npm install mysql2
|
||||
*/
|
||||
// mysql: {
|
||||
// client: 'mysql2',
|
||||
// connection: {
|
||||
// host: env.get('DB_HOST'),
|
||||
// port: env.get('DB_PORT'),
|
||||
// user: env.get('DB_USER'),
|
||||
// password: env.get('DB_PASSWORD'),
|
||||
// database: env.get('DB_DATABASE'),
|
||||
// },
|
||||
// migrations: {
|
||||
// naturalSort: true,
|
||||
// paths: ['database/migrations'],
|
||||
// },
|
||||
// debug: app.inDev,
|
||||
// },
|
||||
|
||||
/**
|
||||
* Microsoft SQL Server connection.
|
||||
* Install package to switch: npm install tedious
|
||||
*/
|
||||
// mssql: {
|
||||
// client: 'mssql',
|
||||
// connection: {
|
||||
// server: env.get('DB_HOST'),
|
||||
// port: env.get('DB_PORT'),
|
||||
// user: env.get('DB_USER'),
|
||||
// password: env.get('DB_PASSWORD'),
|
||||
// database: env.get('DB_DATABASE'),
|
||||
// },
|
||||
// migrations: {
|
||||
// naturalSort: true,
|
||||
// paths: ['database/migrations'],
|
||||
// },
|
||||
// debug: app.inDev,
|
||||
// },
|
||||
|
||||
/**
|
||||
* libSQL (Turso) connection.
|
||||
* Install package to switch: npm install @libsql/client
|
||||
*/
|
||||
// libsql: {
|
||||
// client: 'libsql',
|
||||
// connection: {
|
||||
// url: env.get('LIBSQL_URL'),
|
||||
// authToken: env.get('LIBSQL_AUTH_TOKEN'),
|
||||
// },
|
||||
// useNullAsDefault: true,
|
||||
// migrations: {
|
||||
// naturalSort: true,
|
||||
// paths: ['database/migrations'],
|
||||
// },
|
||||
// debug: app.inDev,
|
||||
// },
|
||||
},
|
||||
})
|
||||
|
||||
|
||||
@@ -1,91 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
/**
|
||||
* Retention for transient pipeline state in Redis. See CONCEPT.md (D11).
|
||||
*/
|
||||
export default {
|
||||
/** How often `gc:scheduler` sweeps. */
|
||||
intervalMs: env.get('GC_INTERVAL_MS', 3 * 60 * 1000),
|
||||
|
||||
/** How often gc:scheduler recomputes the completeness of releases that changed (LOG.md 2026-10-06 (6)). */
|
||||
releaseCompleteIntervalMs: env.get('RELEASE_COMPLETE_INTERVAL_MS', 30 * 60 * 1000),
|
||||
|
||||
/** How often gc:scheduler merges uploads split per poster (D35); it reads every release. */
|
||||
mergeIntervalMs: env.get('GC_MERGE_INTERVAL_MS', 60 * 60 * 1000),
|
||||
|
||||
/** Time per sweep for copying old file keys into file_keys until it's complete (LOG.md 2026-10-06 (7)). */
|
||||
fileKeysFillMs: env.get('FILE_KEYS_FILL_SECONDS', 90) * 1000,
|
||||
|
||||
jobs: {
|
||||
/** Completed jobs are only kept for inspection; keep them briefly. */
|
||||
completedAgeSeconds: env.get('GC_COMPLETED_AGE_SECONDS', 60 * 60),
|
||||
completedMaxCount: 1000,
|
||||
/** Failed jobs are kept longer so they can be inspected and retried. */
|
||||
failedAgeSeconds: env.get('GC_FAILED_AGE_SECONDS', 7 * 24 * 60 * 60),
|
||||
},
|
||||
|
||||
/**
|
||||
* Upper bound on waiting body jobs. Obfuscated traffic can exceed what the
|
||||
* line downloads for hours; beyond this, the oldest waiting jobs are
|
||||
* dropped so Redis stays bounded (~1 KB per job).
|
||||
*/
|
||||
bodyQueueMax: env.get('GC_BODY_QUEUE_MAX', 500_000),
|
||||
|
||||
binaries: {
|
||||
/**
|
||||
* A binary that gets no new segments for this long is deleted: incomplete
|
||||
* ones are given up on, completed ones have been copied into `files`.
|
||||
*/
|
||||
staleAfterSeconds: env.get('GC_BINARY_STALE_SECONDS', 48 * 60 * 60),
|
||||
/**
|
||||
* Header-derived binaries that still have one article after this long:
|
||||
* uploads that give every article its own random name (and poster) never
|
||||
* complete from headers. A real file gets its next articles in minutes.
|
||||
*/
|
||||
singleArticleAfterSeconds: env.get('GC_SINGLE_ARTICLE_SECONDS', 2 * 60 * 60),
|
||||
/**
|
||||
* Completed header-derived binaries without a fingerprint (size, MD5 of
|
||||
* the first 16 KiB): their file exists and no par2 set can name them
|
||||
* later, so they go sooner than staleAfterSeconds.
|
||||
*/
|
||||
unfingerprintedAfterSeconds: env.get('GC_UNFINGERPRINTED_SECONDS', 6 * 60 * 60),
|
||||
/**
|
||||
* A finished binary whose file waits for content inspection is kept this long at
|
||||
* most (it needs the fingerprint inspection stores for par2 naming, D16).
|
||||
*/
|
||||
awaitInspectionSeconds: env.get('GC_AWAIT_INSPECTION_SECONDS', 7 * 24 * 60 * 60),
|
||||
},
|
||||
|
||||
/**
|
||||
* Staged lone segments (CONCEPT.md D35) are dropped after this long. A
|
||||
* real file's next segment comes in the same or the next fetch range,
|
||||
* seconds to minutes later; at backfill speed two hours of waiting
|
||||
* segments were 12M rows, 10 GB.
|
||||
*/
|
||||
stagingTtlSeconds: env.get('STAGING_TTL_SECONDS', 30 * 60),
|
||||
|
||||
/** Releases without files or binaries (their binaries were collected) are deleted after this long. */
|
||||
emptyReleaseAfterSeconds: env.get('GC_EMPTY_RELEASE_SECONDS', 2 * 60 * 60),
|
||||
|
||||
/**
|
||||
* Bulk deletes run in batches of this size, each kind (copied parts,
|
||||
* single-article and unfingerprinted binaries, empty releases) for at most
|
||||
* this long per sweep, so one backlog can't starve the others.
|
||||
*/
|
||||
deleteBatch: 20_000,
|
||||
deleteBudgetMs: env.get('GC_DELETE_BUDGET_MS', 60 * 1000),
|
||||
/** Bloated indexes are rebuilt at most this often (IndexMaintenance). */
|
||||
reindexIntervalMs: env.get('GC_REINDEX_INTERVAL_MS', 7 * 24 * 60 * 60 * 1000),
|
||||
|
||||
/**
|
||||
* Deferred files (CONCEPT.md D28) are deleted this long after they were
|
||||
* first seen, processed or not. The articles stay on the server.
|
||||
*/
|
||||
deferredRetentionDays: env.get('GC_DEFERRED_RETENTION_DAYS', 30),
|
||||
|
||||
/** Redis hashes from the old header worker (before D12); they only need to expire. */
|
||||
legacyPendingFiles: {
|
||||
prefix: 'file:',
|
||||
ttlSeconds: 48 * 60 * 60,
|
||||
},
|
||||
}
|
||||
@@ -1,121 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
export default {
|
||||
/** How often `index:scheduler` checks the groups for new articles. */
|
||||
intervalMs: env.get('INDEX_INTERVAL_MS', 60 * 1000),
|
||||
/** Articles per fetch job (one XOVER call). */
|
||||
batchSize: BigInt(env.get('INDEX_BATCH_SIZE', 20_000)),
|
||||
/** Upper bound on fetch jobs queued per group per tick, so catching up is spread out. */
|
||||
maxBatchesPerTick: env.get('INDEX_MAX_BATCHES_PER_TICK', 5),
|
||||
/**
|
||||
* Where a group that was never indexed starts: this many articles before
|
||||
* the newest one. The server's low-water mark can be years back.
|
||||
*/
|
||||
initialBackfill: BigInt(env.get('INDEX_INITIAL_BACKFILL', 50_000)),
|
||||
|
||||
/**
|
||||
* Obfuscated articles whose message-id matches one of these are not sent
|
||||
* to the body queue: their uploader spreads every file over many groups,
|
||||
* so identifying them would take far more bandwidth than there is
|
||||
* (see LOG.md, 2026-09-28 (9)). Normal-subject articles are unaffected.
|
||||
*/
|
||||
skipBodyMessageIds: [
|
||||
// hex7$hex7$hex7@hex10.hex5, ~60k articles/min over 10+ groups
|
||||
/^<?[0-9a-f]{7}\$[0-9a-f]{7}\$[0-9a-f]{7}@[0-9a-f]{10}\.[0-9a-f]{5}>?$/,
|
||||
],
|
||||
|
||||
/**
|
||||
* Obfuscated articles whose message-id matches one of these don't go to
|
||||
* the body queue. Instead they're recorded per file in `deferred_files`
|
||||
* (their subject keeps the file name, only the part counter is missing).
|
||||
* `worker:deferred` later fetches one body per file (CONCEPT.md D28).
|
||||
*/
|
||||
deferBodyMessageIds: [
|
||||
// ngPost: <hex32@ngPost>. Mostly RAR5 with encrypted headers (LOG.md 2026-09-28 (14)).
|
||||
/^<?[0-9a-f]{32}@ngPost>?$/,
|
||||
],
|
||||
/**
|
||||
* Spotnet header groups (CONCEPT.md D29): their headers are spots, not
|
||||
* binaries. Add one with `group:add free.pt --backfill 5000`.
|
||||
*/
|
||||
spotGroups: ['free.pt'],
|
||||
/**
|
||||
* Spots posted within this many days are imported (their NZB read and its
|
||||
* files added). Older spots are listed: title, size and date, with the NZB
|
||||
* read on first download.
|
||||
*/
|
||||
spotImportDays: env.get('SPOT_IMPORT_DAYS', 30),
|
||||
|
||||
/**
|
||||
* Groups whose lone header segments wait in `staged_segments` until their
|
||||
* file shows a second one (CONCEPT.md D35). Comma-separated; `*` for all.
|
||||
*/
|
||||
stagingGroups: (env.get('STAGING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
|
||||
|
||||
/**
|
||||
* Groups whose staged segments also group by poster, file counter `[i/M]`
|
||||
* and part count (CONCEPT.md D36), for uploads with a random subject per
|
||||
* article. Needs staging. Comma-separated; `*` for all.
|
||||
*/
|
||||
counterGroupingGroups: (env.get('COUNTER_GROUPING_GROUPS', '') ?? '').split(',').map((g) => g.trim()).filter(Boolean),
|
||||
|
||||
/**
|
||||
* Header backfill (`group:backfill`, CONCEPT.md D32): low-priority,
|
||||
* header-only fetch ranges walking back through a group's history.
|
||||
*/
|
||||
seed: {
|
||||
/** NNTP connections the seed API fetches with for relay nodes (CONCEPT.md D38); 0: relay off. */
|
||||
nntpConnections: env.get('SEED_NNTP_CONNECTIONS', 0),
|
||||
/**
|
||||
* Silence that counts as a dead connection during a relay XOVER (an
|
||||
* inactivity timeout: slow ranges keep sending). Chunks that "never
|
||||
* answered" in 10 minutes came in 0.3 s on a fresh connection.
|
||||
*/
|
||||
nntpTimeoutMs: env.get('SEED_NNTP_TIMEOUT_MS', 90 * 1000),
|
||||
/**
|
||||
* Ask the news server for gzip-compressed overviews (SEED_NNTP_COMPRESS=false: plain).
|
||||
* On one connection plain had a shorter tail for 2026 articles, but with 20 relay
|
||||
* connections both gave ~1.5 chunks/s, plain at many times the relay's CPU and
|
||||
* more traffic (LOG.md 2026-10-06 (9)).
|
||||
*/
|
||||
nntpCompress: env.get('SEED_NNTP_COMPRESS', true),
|
||||
},
|
||||
|
||||
backfill: {
|
||||
/** Backfill ranges waiting or running at a time, over all groups. */
|
||||
maxQueued: env.get('BACKFILL_MAX_QUEUED', 40),
|
||||
/** BullMQ priority (higher runs later; live ranges have none, so they go first). */
|
||||
priority: 10,
|
||||
/**
|
||||
* No new backfill ranges while the database's NVMe part (default
|
||||
* tablespace) or its HDD part (tablespace `hdd`, the files table) is
|
||||
* larger than this. Checked every scheduler tick.
|
||||
*/
|
||||
maxDefaultGb: env.get('BACKFILL_MAX_NVME_GB', 45),
|
||||
maxHddGb: env.get('BACKFILL_MAX_HDD_GB', 150),
|
||||
},
|
||||
|
||||
/**
|
||||
* Dropping garbage uploads early (CONCEPT.md D42). A signature (posting tool, first
|
||||
* group) is dropped once at least minProbes of its probed random-named releases are
|
||||
* known and at most maxNamedRate of them got a real name or password; its
|
||||
* random-named files are then not stored, except explorePercent of its releases.
|
||||
*/
|
||||
garbage: {
|
||||
/** Off: nothing is filtered or deleted (the learning still runs). */
|
||||
enabled: env.get('GARBAGE_FILTER', true),
|
||||
explorePercent: env.get('GARBAGE_EXPLORE_PERCENT', 2),
|
||||
minProbes: 100,
|
||||
maxNamedRate: 0.005,
|
||||
/** Probed releases learned from: 1 in this many (by release id), for the last windowDays. */
|
||||
probeSample: 4,
|
||||
windowDays: 30,
|
||||
/** How often gc:scheduler learns the signatures again. */
|
||||
learnIntervalMs: env.get('GARBAGE_LEARN_INTERVAL_MS', 3 * 60 * 60 * 1000),
|
||||
},
|
||||
|
||||
/** worker:deferred leaves a file alone until it has had no new articles for this long. */
|
||||
deferredSettleMs: env.get('DEFERRED_SETTLE_MS', 15 * 60 * 1000),
|
||||
/** Files of at most this many parts are downloaded completely (small archives, NZBs, par2 index files). */
|
||||
deferredExpandMaxParts: env.get('DEFERRED_EXPAND_MAX_PARTS', 8),
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
export default {
|
||||
host: env.get('NNTP_HOST'),
|
||||
port: env.get('NNTP_PORT', 119),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', false),
|
||||
/** Connections per process. Mind the provider's limit across all processes. */
|
||||
poolSize: env.get('NNTP_POOL_SIZE', 10),
|
||||
/** A command without an answer this long counts as a dead connection (XOVER of 20k headers takes ~0.2 s). */
|
||||
commandTimeoutMs: env.get('NNTP_COMMAND_TIMEOUT_MS', 120_000),
|
||||
/** TLS certificate hostname when NNTP_HOST is a tunnel (seeding nodes; lib/nntp/xover_client). */
|
||||
tlsServername: env.get('NNTP_TLS_SERVERNAME'),
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
export default {
|
||||
connection: {
|
||||
host: env.get('REDIS_HOST', '127.0.0.1'),
|
||||
port: env.get('REDIS_PORT', 6379),
|
||||
},
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
import env from '#start/env'
|
||||
|
||||
export default {
|
||||
connection: 'main',
|
||||
connections: {
|
||||
main: {
|
||||
host: env.get('REDIS_HOST', '127.0.0.1'),
|
||||
port: env.get('REDIS_PORT'),
|
||||
password: env.get('REDIS_PASSWORD')?.release(),
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
import { defineConfig } from '@adonisjs/static'
|
||||
|
||||
/**
|
||||
* Configuration options to tweak the static files middleware.
|
||||
* The complete set of options are documented on the
|
||||
* official documentation website.
|
||||
*
|
||||
* https://docs.adonisjs.com/guides/static-assets
|
||||
*/
|
||||
const staticServerConfig = defineConfig({
|
||||
enabled: true,
|
||||
etag: true,
|
||||
lastModified: true,
|
||||
dotFiles: 'ignore',
|
||||
})
|
||||
|
||||
export default staticServerConfig
|
||||
@@ -1,21 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'groups'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id')
|
||||
table.string('name').notNullable().unique()
|
||||
table.boolean('active').notNullable().defaultTo(true)
|
||||
table.bigInteger('last_indexed_id').defaultTo(0)
|
||||
|
||||
table.timestamp('created_at', { useTz: true })
|
||||
table.timestamp('updated_at', { useTz: true })
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'files'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.increments('id').primary()
|
||||
table.string('filename').notNullable()
|
||||
table.string('poster').notNullable()
|
||||
table.bigInteger('date').notNullable()
|
||||
table.integer('parts').notNullable()
|
||||
table.jsonb('message_ids').notNullable()
|
||||
table.jsonb('groups')
|
||||
|
||||
table.timestamp('created_at', { useTz: true })
|
||||
table.timestamp('updated_at', { useTz: true })
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,32 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Files that are still being collected from XOVER. One row per uploaded file,
|
||||
* identified by file_key (see lib/subject/file_key.ts).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'binaries'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.string('file_key', 40).notNullable().unique()
|
||||
table.text('poster').notNullable()
|
||||
table.text('subject').notNullable()
|
||||
table.text('name').notNullable()
|
||||
table.integer('total_parts').notNullable()
|
||||
table.jsonb('groups').notNullable()
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.timestamp('first_seen_at', { useTz: true }).notNullable()
|
||||
table.timestamp('last_seen_at', { useTz: true }).notNullable().index()
|
||||
table.timestamp('completed_at', { useTz: true })
|
||||
// Set when two different articles claim the same part number (see IngestService).
|
||||
table.timestamp('conflicted_at', { useTz: true })
|
||||
table.integer('file_id').unsigned().references('files.id').onDelete('SET NULL')
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,24 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Segments (articles) of a binary. The primary key makes re-ingesting the
|
||||
* same range a no-op.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'parts'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.bigInteger('binary_id').notNullable().references('binaries.id').onDelete('CASCADE')
|
||||
table.integer('number').notNullable()
|
||||
table.text('message_id').notNullable()
|
||||
table.integer('bytes').notNullable()
|
||||
table.bigInteger('article_number')
|
||||
table.primary(['binary_id', 'number'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'files'
|
||||
|
||||
async up() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.string('file_key', 40).unique()
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable(this.tableName, (table) => {
|
||||
table.dropColumn('file_key')
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* A release is one download: the files of one post (archive volumes, par2
|
||||
* set, nfo). See lib/release/release_key.ts and CONCEPT.md (D16).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
protected tableName = 'releases'
|
||||
|
||||
async up() {
|
||||
this.schema.createTable(this.tableName, (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.string('release_key', 40).notNullable().unique()
|
||||
table.text('name').notNullable()
|
||||
// Null for obfuscated posts, whose poster is random per article.
|
||||
table.text('poster')
|
||||
// Number of files announced by the [i/M] subject counter, if any.
|
||||
table.integer('file_total')
|
||||
table.jsonb('groups').notNullable()
|
||||
table.timestamp('posted_at', { useTz: true })
|
||||
table.timestamp('first_seen_at', { useTz: true }).notNullable()
|
||||
table.timestamp('last_seen_at', { useTz: true }).notNullable()
|
||||
})
|
||||
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
table.bigInteger('release_id').references('releases.id').onDelete('SET NULL').index()
|
||||
})
|
||||
this.schema.alterTable('files', (table) => {
|
||||
table.bigInteger('release_id').references('releases.id').onDelete('SET NULL').index()
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.alterTable('files', (table) => table.dropColumn('release_id'))
|
||||
this.schema.alterTable('binaries', (table) => table.dropColumn('release_id'))
|
||||
this.schema.dropTable(this.tableName)
|
||||
}
|
||||
}
|
||||
@@ -1,33 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Par2-based grouping of obfuscated posts (CONCEPT.md D16): the par2 file
|
||||
* list of a release, and the exact file size of body-derived binaries to
|
||||
* match against it.
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('binaries', (table) => {
|
||||
// Exact file size from =ybegin; only known for body-derived binaries.
|
||||
table.bigInteger('size').index()
|
||||
})
|
||||
this.schema.alterTable('releases', (table) => {
|
||||
// How the release was recognised: subject, stem, weak-stem or par2.
|
||||
table.string('source', 16)
|
||||
})
|
||||
this.schema.createTable('release_files', (table) => {
|
||||
table.bigInteger('release_id').notNullable().references('releases.id').onDelete('CASCADE')
|
||||
table.string('par2_file_id', 32).notNullable()
|
||||
table.text('name').notNullable()
|
||||
table.bigInteger('size').notNullable().index()
|
||||
table.string('md5_16k', 32).notNullable()
|
||||
table.primary(['release_id', 'par2_file_id'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('release_files')
|
||||
this.schema.alterTable('releases', (table) => table.dropColumn('source'))
|
||||
this.schema.alterTable('binaries', (table) => table.dropColumn('size'))
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||
|
||||
/**
|
||||
* Article ranges handed to worker:fetch, so a range only counts as indexed
|
||||
* once it was fetched successfully (CONCEPT.md D18).
|
||||
*/
|
||||
export default class extends BaseSchema {
|
||||
async up() {
|
||||
this.schema.alterTable('groups', (table) => {
|
||||
// How far the scheduler has queued; last_indexed_id becomes the
|
||||
// watermark below which every article was fetched.
|
||||
table.bigInteger('last_queued_id').notNullable().defaultTo(0)
|
||||
})
|
||||
this.defer(async (db) => {
|
||||
await db.rawQuery('UPDATE groups SET last_queued_id = last_indexed_id')
|
||||
})
|
||||
|
||||
this.schema.createTable('fetch_ranges', (table) => {
|
||||
table.bigIncrements('id')
|
||||
table.integer('group_id').notNullable().references('groups.id').onDelete('CASCADE')
|
||||
table.bigInteger('start_id').notNullable()
|
||||
table.bigInteger('end_id').notNullable()
|
||||
// queued | done | failed
|
||||
table.string('status', 8).notNullable().defaultTo('queued')
|
||||
table.integer('requeues').notNullable().defaultTo(0)
|
||||
table.text('error')
|
||||
table.timestamp('created_at', { useTz: true }).notNullable()
|
||||
table.timestamp('updated_at', { useTz: true }).notNullable()
|
||||
table.index(['group_id', 'status'])
|
||||
})
|
||||
}
|
||||
|
||||
async down() {
|
||||
this.schema.dropTable('fetch_ranges')
|
||||
this.schema.alterTable('groups', (table) => table.dropColumn('last_queued_id'))
|
||||
}
|
||||
}
|
||||
Loaded 100 of 264 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user