Compare commits
2
Commits
44c0c73b05
..
master
| Author | SHA1 | Date | |
|---|---|---|---|
 Daan MeijerandClaude Opus 5.5
|
7880633467 | ||
|
Daan MeijerandClaude Opus 5.5
|
7c43abd230 |
No files matched your search
Vendored
+5
-1
@@ -5,6 +5,10 @@
|
||||
|
||||
/// <reference path="../../adonisrc.ts" />
|
||||
/// <reference path="../../config/auth.ts" />
|
||||
/// <reference path="../../config/coordinator.ts" />
|
||||
/// <reference path="../../config/gc.ts" />
|
||||
/// <reference path="../../config/hash.ts" />
|
||||
/// <reference path="../../config/indexer.ts" />
|
||||
/// <reference path="../../config/logger.ts" />
|
||||
/// <reference path="../../config/nntp.ts" />
|
||||
/// <reference path="../../config/queue.ts" />
|
||||
/// <reference path="../../config/redis.ts" />
|
||||
@@ -30,54 +30,6 @@ const routes = {
|
||||
tokens: [{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],
|
||||
types: placeholder as Registry['profile.access_tokens.destroy']['types'],
|
||||
},
|
||||
'releases.index': {
|
||||
methods: ["GET","HEAD"],
|
||||
pattern: '/api/v1/releases',
|
||||
tokens: [{"old":"/api/v1/releases","type":0,"val":"api","end":""},{"old":"/api/v1/releases","type":0,"val":"v1","end":""},{"old":"/api/v1/releases","type":0,"val":"releases","end":""}],
|
||||
types: placeholder as Registry['releases.index']['types'],
|
||||
},
|
||||
'releases.nzb': {
|
||||
methods: ["GET","HEAD"],
|
||||
pattern: '/api/v1/releases/:id/nzb',
|
||||
tokens: [{"old":"/api/v1/releases/:id/nzb","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id/nzb","type":1,"val":"id","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"nzb","end":""}],
|
||||
types: placeholder as Registry['releases.nzb']['types'],
|
||||
},
|
||||
'node.node.lease': {
|
||||
methods: ["POST"],
|
||||
pattern: '/api/v1/node/lease',
|
||||
tokens: [{"old":"/api/v1/node/lease","type":0,"val":"api","end":""},{"old":"/api/v1/node/lease","type":0,"val":"v1","end":""},{"old":"/api/v1/node/lease","type":0,"val":"node","end":""},{"old":"/api/v1/node/lease","type":0,"val":"lease","end":""}],
|
||||
types: placeholder as Registry['node.node.lease']['types'],
|
||||
},
|
||||
'node.node.stream': {
|
||||
methods: ["GET","HEAD"],
|
||||
pattern: '/api/v1/node/ranges/:id/stream',
|
||||
tokens: [{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/stream","type":0,"val":"stream","end":""}],
|
||||
types: placeholder as Registry['node.node.stream']['types'],
|
||||
},
|
||||
'node.node.batch': {
|
||||
methods: ["POST"],
|
||||
pattern: '/api/v1/node/ranges/:id/batches',
|
||||
tokens: [{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/batches","type":0,"val":"batches","end":""}],
|
||||
types: placeholder as Registry['node.node.batch']['types'],
|
||||
},
|
||||
'node.node.heartbeat': {
|
||||
methods: ["POST"],
|
||||
pattern: '/api/v1/node/ranges/:id/heartbeat',
|
||||
tokens: [{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/heartbeat","type":0,"val":"heartbeat","end":""}],
|
||||
types: placeholder as Registry['node.node.heartbeat']['types'],
|
||||
},
|
||||
'node.node.finish': {
|
||||
methods: ["POST"],
|
||||
pattern: '/api/v1/node/ranges/:id/finish',
|
||||
tokens: [{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/finish","type":0,"val":"finish","end":""}],
|
||||
types: placeholder as Registry['node.node.finish']['types'],
|
||||
},
|
||||
'node.node.release': {
|
||||
methods: ["POST"],
|
||||
pattern: '/api/v1/node/ranges/:id/release',
|
||||
tokens: [{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"api","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"v1","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"node","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"ranges","end":""},{"old":"/api/v1/node/ranges/:id/release","type":1,"val":"id","end":""},{"old":"/api/v1/node/ranges/:id/release","type":0,"val":"release","end":""}],
|
||||
types: placeholder as Registry['node.node.release']['types'],
|
||||
},
|
||||
} as const satisfies Record<string, AdonisEndpoint>
|
||||
|
||||
export { routes }
|
||||
|
||||
-96
@@ -55,100 +55,4 @@ export interface Registry {
|
||||
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/access_tokens_controller').default['destroy']>>>
|
||||
}
|
||||
}
|
||||
'releases.index': {
|
||||
methods: ["GET","HEAD"]
|
||||
pattern: '/api/v1/releases'
|
||||
types: {
|
||||
body: {}
|
||||
paramsTuple: []
|
||||
params: {}
|
||||
query: {}
|
||||
response: ExtractResponse<Awaited<ReturnType<import('#controllers/releases_controller').default['index']>>>
|
||||
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/releases_controller').default['index']>>>
|
||||
}
|
||||
}
|
||||
'releases.nzb': {
|
||||
methods: ["GET","HEAD"]
|
||||
pattern: '/api/v1/releases/:id/nzb'
|
||||
types: {
|
||||
body: {}
|
||||
paramsTuple: [ParamValue]
|
||||
params: { id: ParamValue }
|
||||
query: {}
|
||||
response: ExtractResponse<Awaited<ReturnType<import('#controllers/releases_controller').default['nzb']>>>
|
||||
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/releases_controller').default['nzb']>>>
|
||||
}
|
||||
}
|
||||
'node.node.lease': {
|
||||
methods: ["POST"]
|
||||
pattern: '/api/v1/node/lease'
|
||||
types: {
|
||||
body: {}
|
||||
paramsTuple: []
|
||||
params: {}
|
||||
query: {}
|
||||
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['lease']>>>
|
||||
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['lease']>>>
|
||||
}
|
||||
}
|
||||
'node.node.stream': {
|
||||
methods: ["GET","HEAD"]
|
||||
pattern: '/api/v1/node/ranges/:id/stream'
|
||||
types: {
|
||||
body: {}
|
||||
paramsTuple: [ParamValue]
|
||||
params: { id: ParamValue }
|
||||
query: {}
|
||||
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['stream']>>>
|
||||
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['stream']>>>
|
||||
}
|
||||
}
|
||||
'node.node.batch': {
|
||||
methods: ["POST"]
|
||||
pattern: '/api/v1/node/ranges/:id/batches'
|
||||
types: {
|
||||
body: {}
|
||||
paramsTuple: [ParamValue]
|
||||
params: { id: ParamValue }
|
||||
query: {}
|
||||
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['batch']>>>
|
||||
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['batch']>>>
|
||||
}
|
||||
}
|
||||
'node.node.heartbeat': {
|
||||
methods: ["POST"]
|
||||
pattern: '/api/v1/node/ranges/:id/heartbeat'
|
||||
types: {
|
||||
body: {}
|
||||
paramsTuple: [ParamValue]
|
||||
params: { id: ParamValue }
|
||||
query: {}
|
||||
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['heartbeat']>>>
|
||||
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['heartbeat']>>>
|
||||
}
|
||||
}
|
||||
'node.node.finish': {
|
||||
methods: ["POST"]
|
||||
pattern: '/api/v1/node/ranges/:id/finish'
|
||||
types: {
|
||||
body: {}
|
||||
paramsTuple: [ParamValue]
|
||||
params: { id: ParamValue }
|
||||
query: {}
|
||||
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['finish']>>>
|
||||
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['finish']>>>
|
||||
}
|
||||
}
|
||||
'node.node.release': {
|
||||
methods: ["POST"]
|
||||
pattern: '/api/v1/node/ranges/:id/release'
|
||||
types: {
|
||||
body: {}
|
||||
paramsTuple: [ParamValue]
|
||||
params: { id: ParamValue }
|
||||
query: {}
|
||||
response: ExtractResponse<Awaited<ReturnType<import('#controllers/node_controller').default['release']>>>
|
||||
errorResponse: ExtractErrorResponse<Awaited<ReturnType<import('#controllers/node_controller').default['release']>>>
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
-14
@@ -18,18 +18,4 @@ export interface ApiDefinition {
|
||||
destroy: typeof routes['profile.access_tokens.destroy']
|
||||
}
|
||||
}
|
||||
releases: {
|
||||
index: typeof routes['releases.index']
|
||||
nzb: typeof routes['releases.nzb']
|
||||
}
|
||||
node: {
|
||||
node: {
|
||||
lease: typeof routes['node.node.lease']
|
||||
stream: typeof routes['node.node.stream']
|
||||
batch: typeof routes['node.node.batch']
|
||||
heartbeat: typeof routes['node.node.heartbeat']
|
||||
finish: typeof routes['node.node.finish']
|
||||
release: typeof routes['node.node.release']
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@ export const controllers = {
|
||||
AccessTokens: () => import('#controllers/access_tokens_controller'),
|
||||
NewAccount: () => import('#controllers/new_account_controller'),
|
||||
Newznab: () => import('#controllers/newznab_controller'),
|
||||
Node: () => import('#controllers/node_controller'),
|
||||
Profile: () => import('#controllers/profile_controller'),
|
||||
Releases: () => import('#controllers/releases_controller'),
|
||||
Seed: () => import('#controllers/seed_controller'),
|
||||
}
|
||||
Vendored
+6
-24
@@ -4,51 +4,33 @@ type ParamValue = string | number | bigint | boolean
|
||||
|
||||
export type ScannedRoutes = {
|
||||
ALL: {
|
||||
'newznab': { paramsTuple?: []; params?: {} }
|
||||
'releases.index': { paramsTuple?: []; params?: {} }
|
||||
'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.stats': { paramsTuple?: []; params?: {} }
|
||||
'auth.new_account.store': { paramsTuple?: []; params?: {} }
|
||||
'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
|
||||
'profile.profile.show': { paramsTuple?: []; params?: {} }
|
||||
'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
|
||||
'releases.index': { paramsTuple?: []; params?: {} }
|
||||
'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.stats': { paramsTuple?: []; params?: {} }
|
||||
'node.node.lease': { paramsTuple?: []; params?: {} }
|
||||
'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'node.node.batch': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
}
|
||||
GET: {
|
||||
'newznab': { paramsTuple?: []; params?: {} }
|
||||
'profile.profile.show': { paramsTuple?: []; params?: {} }
|
||||
'releases.index': { paramsTuple?: []; params?: {} }
|
||||
'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.stats': { paramsTuple?: []; params?: {} }
|
||||
'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'profile.profile.show': { paramsTuple?: []; params?: {} }
|
||||
}
|
||||
HEAD: {
|
||||
'newznab': { paramsTuple?: []; params?: {} }
|
||||
'profile.profile.show': { paramsTuple?: []; params?: {} }
|
||||
'releases.index': { paramsTuple?: []; params?: {} }
|
||||
'releases.show': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.nzb': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'releases.stats': { paramsTuple?: []; params?: {} }
|
||||
'node.node.stream': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'profile.profile.show': { paramsTuple?: []; params?: {} }
|
||||
}
|
||||
POST: {
|
||||
'auth.new_account.store': { paramsTuple?: []; params?: {} }
|
||||
'auth.access_tokens.store': { paramsTuple?: []; params?: {} }
|
||||
'profile.access_tokens.destroy': { paramsTuple?: []; params?: {} }
|
||||
'node.node.lease': { paramsTuple?: []; params?: {} }
|
||||
'node.node.batch': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'node.node.heartbeat': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'node.node.finish': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'node.node.release': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
'node.node.corrupt': { paramsTuple: [ParamValue]; params: {'id': ParamValue} }
|
||||
}
|
||||
}
|
||||
declare module '@adonisjs/core/types/http' {
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
{"root":[{"domain":"root","pattern":"/api/v1/releases","matchers":{},"tokens":[{"old":"/api/v1/releases","type":0,"val":"api","end":""},{"old":"/api/v1/releases","type":0,"val":"v1","end":""},{"old":"/api/v1/releases","type":0,"val":"releases","end":""}],"meta":{"params":[]},"name":"releases.index","handler":{"method":"index","reference":[null,"index"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id","type":1,"val":"id","end":"","matcher":{}}],"meta":{"params":["id"]},"name":"releases.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/releases/:id/nzb","matchers":{"id":{"match":{}}},"tokens":[{"old":"/api/v1/releases/:id/nzb","type":0,"val":"api","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"v1","end":""},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"releases","end":""},{"old":"/api/v1/releases/:id/nzb","type":1,"val":"id","end":"","matcher":{}},{"old":"/api/v1/releases/:id/nzb","type":0,"val":"nzb","end":""}],"meta":{"params":["id"]},"name":"releases.nzb","handler":{"method":"nzb","reference":[null,"nzb"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/stats","matchers":{},"tokens":[{"old":"/api/v1/stats","type":0,"val":"api","end":""},{"old":"/api/v1/stats","type":0,"val":"v1","end":""},{"old":"/api/v1/stats","type":0,"val":"stats","end":""}],"meta":{"params":[]},"name":"releases.stats","handler":{"method":"stats","reference":[null,"stats"],"importExpression":"()=>import('#controllers/releases_controller')","name":"ReleasesController"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/signup","matchers":{},"tokens":[{"old":"/api/v1/auth/signup","type":0,"val":"api","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/signup","type":0,"val":"signup","end":""}],"meta":{"params":[]},"name":"auth.new_account.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/new_account_controller')","name":"NewAccount"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/auth/login","matchers":{},"tokens":[{"old":"/api/v1/auth/login","type":0,"val":"api","end":""},{"old":"/api/v1/auth/login","type":0,"val":"v1","end":""},{"old":"/api/v1/auth/login","type":0,"val":"auth","end":""},{"old":"/api/v1/auth/login","type":0,"val":"login","end":""}],"meta":{"params":[]},"name":"auth.access_tokens.store","handler":{"method":"store","reference":[null,"store"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/profile","matchers":{},"tokens":[{"old":"/api/v1/account/profile","type":0,"val":"api","end":""},{"old":"/api/v1/account/profile","type":0,"val":"v1","end":""},{"old":"/api/v1/account/profile","type":0,"val":"account","end":""},{"old":"/api/v1/account/profile","type":0,"val":"profile","end":""}],"meta":{"params":[]},"name":"profile.profile.show","handler":{"method":"show","reference":[null,"show"],"importExpression":"()=>import('#controllers/profile_controller')","name":"Profile"},"methods":["GET","HEAD"],"middleware":{}},{"domain":"root","pattern":"/api/v1/account/logout","matchers":{},"tokens":[{"old":"/api/v1/account/logout","type":0,"val":"api","end":""},{"old":"/api/v1/account/logout","type":0,"val":"v1","end":""},{"old":"/api/v1/account/logout","type":0,"val":"account","end":""},{"old":"/api/v1/account/logout","type":0,"val":"logout","end":""}],"meta":{"params":[]},"name":"profile.access_tokens.destroy","handler":{"method":"destroy","reference":[null,"destroy"],"importExpression":"()=>import('#controllers/access_tokens_controller')","name":"AccessTokens"},"methods":["POST"],"middleware":{}}]}
|
||||
+7
-2
@@ -1,6 +1,11 @@
|
||||
node_modules
|
||||
build
|
||||
tmp
|
||||
.env
|
||||
debug
|
||||
.git
|
||||
edge/target
|
||||
.idea
|
||||
.vscode
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
*.log
|
||||
+3
-8
@@ -12,14 +12,9 @@ APP_URL=http://${HOST}:${PORT}
|
||||
# Session
|
||||
SESSION_DRIVER=cookie
|
||||
|
||||
# Database (the rebuild's own)
|
||||
DB_HOST=127.0.0.1
|
||||
DB_PORT=5432
|
||||
DB_USER=usenet_indexer
|
||||
DB_PASSWORD=
|
||||
DB_DATABASE=usenet_rebuild
|
||||
|
||||
#--------------------------------------------------------------------
|
||||
# CORS (configure allowed origins for API access)
|
||||
#--------------------------------------------------------------------
|
||||
# CORS_ORIGIN=http://localhost:5173,http://localhost:3000
|
||||
# CORS_ORIGIN=http://localhost:5173,http://localhost:3000
|
||||
# Newznab API key (/api?apikey=...); leave empty to allow access without a key
|
||||
NEWZNAB_API_KEY=
|
||||
@@ -1,12 +1,8 @@
|
||||
# Tests every push; a push to rebuild (voyager 2) that passes is deployed to
|
||||
# homelab.
|
||||
#
|
||||
# Deploying: the job SSHes to homelab with DEPLOY_SSH_KEY. That key may only
|
||||
# run ~/bin/deploy-usenet-indexer (a forced command in authorized_keys),
|
||||
# which passes "deploy-rebuild <sha>" on to ~/bin/deploy-usenet-rebuild
|
||||
# (deploy/deploy.sh): it fetches the commit from this repo, builds the
|
||||
# images, runs the migrations and recreates the app services (never
|
||||
# postgres). See deploy/README.md.
|
||||
# Tests every push. Voyager 1 (this branch) is retired since 2026-10-11: its
|
||||
# database was deleted on 2026-10-09, and a push to master no longer deploys.
|
||||
# Voyager 2 (branch rebuild) has its own workflow and deploy
|
||||
# (deploy/README.md there); homelab's ~/bin/deploy-usenet-indexer refuses
|
||||
# voyager 1 deploys too.
|
||||
name: Test and deploy
|
||||
|
||||
on:
|
||||
@@ -59,39 +55,10 @@ jobs:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- name: Rust toolchain
|
||||
run: |
|
||||
command -v cargo || curl -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
# The edge worker (edge/, Rust): built first, since the e2e suite runs edge/target/release/edge-node against
|
||||
# the node API; then its own tests, and the Rust and TypeScript RangeIndexers compared on the committed
|
||||
# captures (edge/fixtures).
|
||||
- name: Build the edge worker
|
||||
run: cargo build --release --manifest-path edge/Cargo.toml --bin edge-node
|
||||
- name: Type check
|
||||
run: npx tsc --noEmit
|
||||
- name: Migrate
|
||||
run: node ace migration:run --force
|
||||
- name: Test
|
||||
run: node ace test
|
||||
- name: cargo test
|
||||
run: cargo test --release --manifest-path edge/Cargo.toml
|
||||
- name: The Rust indexer over the fixtures
|
||||
run: edge/bench-fixtures.sh
|
||||
|
||||
deploy:
|
||||
needs: [test]
|
||||
if: github.ref == 'refs/heads/rebuild'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Deploy to homelab
|
||||
env:
|
||||
DEPLOY_SSH_KEY: ${{ secrets.DEPLOY_SSH_KEY }}
|
||||
DEPLOY_KNOWN_HOSTS: ${{ secrets.DEPLOY_KNOWN_HOSTS }}
|
||||
run: |
|
||||
install -m 700 -d ~/.ssh
|
||||
printf '%s\n' "$DEPLOY_SSH_KEY" > ~/.ssh/deploy_key
|
||||
chmod 600 ~/.ssh/deploy_key
|
||||
printf '%s\n' "$DEPLOY_KNOWN_HOSTS" > ~/.ssh/known_hosts
|
||||
ssh -i ~/.ssh/deploy_key -o IdentitiesOnly=yes -o BatchMode=yes \
|
||||
daan@10.206.0.175 "deploy-rebuild ${{ github.sha }}"
|
||||
@@ -24,6 +24,3 @@ yarn-error.log
|
||||
|
||||
# Platform specific
|
||||
.DS_Store
|
||||
|
||||
# Generated by the dev tooling
|
||||
.adonisjs/server/routes.json
|
||||
+14
-30
@@ -1,25 +1,13 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
#
|
||||
# Voyager 2's images (deploy/compose.yaml):
|
||||
# --target app the AdonisJS app: web server, coordinator, migrations, ace commands
|
||||
# --target edge the Rust edge worker (edge/, bin edge-node)
|
||||
|
||||
# Node is pinned as in CI (.nvmrc): on 24.21, jsonschema 1.5.0 (used by Ace to
|
||||
# validate commands) throws "Invalid URL" on $ref, so no ace command starts.
|
||||
# Node is pinned: on 24.21, jsonschema 1.5.0 (used by Ace to validate commands)
|
||||
# throws "Invalid URL" on $ref, so no ace command starts. See LOG.md.
|
||||
ARG NODE_VERSION=24.16
|
||||
|
||||
# The Rust worker (before the app: its image carries edge-bench). Registry and target dir are build caches, so a rebuild only compiles what changed.
|
||||
FROM rust:1-bookworm AS edge-build
|
||||
WORKDIR /src
|
||||
COPY edge ./
|
||||
RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/src/target \
|
||||
cargo build --release --locked --bin edge-node --bin edge-bench \
|
||||
&& cp target/release/edge-node target/release/edge-bench /usr/local/bin/
|
||||
|
||||
# The app, compiled, with production dependencies only.
|
||||
FROM node:${NODE_VERSION}-bookworm AS app-build
|
||||
WORKDIR /src
|
||||
# Build stage: dev dependencies, TypeScript compile, native addons.
|
||||
# The full node image ships python3/make/g++, needed to compile yencode.
|
||||
FROM node:${NODE_VERSION}-bookworm AS build
|
||||
WORKDIR /app
|
||||
COPY package.json package-lock.json ./
|
||||
RUN npm ci
|
||||
COPY . .
|
||||
@@ -27,19 +15,15 @@ RUN node ace build \
|
||||
&& cd build \
|
||||
&& npm ci --omit=dev
|
||||
|
||||
FROM node:${NODE_VERSION}-bookworm-slim AS app
|
||||
# Runtime stage: compiled app with production dependencies only.
|
||||
FROM node:${NODE_VERSION}-bookworm-slim
|
||||
ENV NODE_ENV=production
|
||||
# bsdtar (libarchive) unpacks compressed RAR entries, e.g. NZBs posted inside RARs.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends libarchive-tools \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
WORKDIR /app
|
||||
COPY --from=app-build /src/build ./
|
||||
# The Rust indexer's bench: `node ace replay:corpus` runs a group's corpus through it (the replay harness).
|
||||
COPY --from=edge-build /usr/local/bin/edge-bench /usr/local/bin/edge-bench
|
||||
COPY --from=build /app/build ./
|
||||
USER node
|
||||
# Every service picks its own command, e.g. `node ace coordinator:run`.
|
||||
# Every service picks its own command, e.g. `node ace worker:fetch`.
|
||||
CMD ["node", "ace", "list"]
|
||||
|
||||
FROM debian:bookworm-slim AS edge
|
||||
COPY --from=edge-build /usr/local/bin/edge-node /usr/local/bin/edge-node
|
||||
# edge-bench replays the corpus for the grouping sweeps (deploy/corpus_sweep.py).
|
||||
COPY --from=edge-build /usr/local/bin/edge-bench /usr/local/bin/edge-bench
|
||||
USER 1000:1000
|
||||
ENTRYPOINT ["/usr/local/bin/edge-node"]
|
||||
-959
@@ -1,959 +0,0 @@
|
||||
# Voyager 2: header-first indexer (concept)
|
||||
|
||||
The concept for what runs (since 2026-10-11; it started as the spec of a
|
||||
rebuild, then a draft for approval). Voyager 1's CONCEPT.md and its
|
||||
decisions D1–D42 are superseded; its domain model still holds, and its
|
||||
features not yet here are the phase 2 backlog below.
|
||||
|
||||
Branch `rebuild`; its successor is designed under *Voyager 3* below (branch
|
||||
`voyager-3`). Phase 1 finds releases from headers and makes their NZBs.
|
||||
Everything learned from articles (names from yEnc headers, par2, archive
|
||||
listings, encryption, spots) is phase 2 and adds to what phase 1 stored; it
|
||||
never rewrites it. Sources: CONCEPT.md, LOG.md 2026-10-08 (13)–(14).
|
||||
|
||||
## Status (2026-10-09)
|
||||
|
||||
Built on this branch (a fresh AdonisJS 7 app; phase 1 code imported from
|
||||
the current app by pipeline stage): coordinator (planner, NNTP pool,
|
||||
prefetch spool, leases, rule sets, fragment merger, settling), the node API
|
||||
(lease, framed stream, batches, finish/release/heartbeat), the edge worker
|
||||
(`edge:node`, `--processes`), the catalog and chunk store, NZBs and a name
|
||||
search. Measured on the workstation (4 cores, 100 Mbit, Eweka):
|
||||
|
||||
- Edge worker, one core: ~210–215 k headers/s on recent a.b.boneless,
|
||||
~330–340 k on a 2013 a.b.ath range; ~0.7–0.8 GB peak per process.
|
||||
(Start: 73–103 k/s, 1.0–1.2 GB.) What's left is spread out (the
|
||||
per-file key ~12%, GC ~10%, zlib ~10%, the scan ~6%).
|
||||
- NNTP: Eweka takes ~1.5–2 s per XOVER command; ~22–30 k lines/s per
|
||||
connection in chunks of 100 k when the line allows; connections don't
|
||||
slow each other.
|
||||
- Bandwidth: compressed XOVER is ~69 B per article on recent boneless
|
||||
(100 k-article chunks of ~7 MB). The workstation's 100 Mbit line carries
|
||||
~140–150 k articles/s, which is where its end-to-end runs now stop (10
|
||||
connections 90–99% busy, ~89 Mbit/s).
|
||||
- End to end at that rate (one 4-core machine): workers about half busy
|
||||
each; server side ~17% (coordinator) + ~8% (web) + ~3% (Postgres) of a
|
||||
core. (Before this round's fixes Postgres was at 32%: the merger's
|
||||
neighbour query read every range per file.)
|
||||
- On homelab (12 cores, 1 Gbit; 2026-10-09, separate containers and
|
||||
database, `/mnt/bulk/usenet-rebuild`): 20 connections fetched ~630 k
|
||||
articles/s (~31 k per connection, ~3 s per 100 k chunk, ~250 Mbit/s at
|
||||
~50 B per article on older boneless) and 8 worker processes indexed all of
|
||||
it, waiting for the spool (~366 M headers in ~12 minutes). Server side at
|
||||
that rate: coordinator 0.4–0.5 cores (1.3–1.7 before it stopped
|
||||
inflating replies to check them), web 0.3–0.45, Postgres (on the HDD)
|
||||
under 0.15. Worker per core there: ~195 k headers/s (boneless 2026),
|
||||
~275 k (ath 2013).
|
||||
- Storage: chunks 17.9–20.3 B per segment (format v2); a release ~1.1 KB.
|
||||
- For the whole target (528 B overview lines): ~36 TB of compressed XOVER
|
||||
to download. At 50 connections (~1–1.25 M lines/s, ~0.6–0.7 Gbit/s)
|
||||
~5–6 days, ~5–6 worker cores, two coordinator shards (one process
|
||||
handles ~60 MB/s), under a core for web and Postgres. The line is the
|
||||
limit below ~0.7 Gbit/s: at 1 Gbit, ~3.5 days at best.
|
||||
|
||||
Where workers run: total speed is set by the account's NNTP connections
|
||||
(~30 k articles/s each), not by worker CPU, so remote workers don't make
|
||||
it faster; they take CPU off homelab (at 1.25 M/s ~7 worker cores) at the
|
||||
cost of relaying the compressed XOVER (≤ ~0.7 Gbit/s up, homelab has 1
|
||||
Gbit/s). Plan: coordinator, web, Postgres and a few workers on homelab,
|
||||
most workers on batcave, a few on poweredge-kantoor with
|
||||
`edge:node --max-mbit` (e.g. 50 of its 160 Mbit/s ≈ 100–125 k articles/s;
|
||||
the cap is shared by the node's processes).
|
||||
|
||||
Every group is tiled from article 1 (2026-10-28): one "ancient" range,
|
||||
status `ancient`, covers 1 … low watermark − 1, what had expired on the
|
||||
server before the group was planned. Never leased, fetched or filled —
|
||||
nothing selects it — and the status panel reports it apart ("Ancient: …
|
||||
articles below the low watermark"); migration 6700 added it for the
|
||||
groups planned before, `Planner.historyRows` for groups planned since.
|
||||
|
||||
Also built (2026-10-09): the live tail (new articles planned every 2
|
||||
minutes from one LIST ACTIVE per name prefix, at priority −1: leased and
|
||||
fetched before history; since 2026-10-27 a round grows the group's open
|
||||
tail range, a planned one at the tail's priority nothing spooled or
|
||||
took, up to RANGE_ARTICLES before it adds rows, as a tail nothing leased
|
||||
had become a sliver a group a round, 127k rows of a median 14k articles;
|
||||
`ranges:merge-tail` packs what such rounds left); the catalog API,
|
||||
Newznab (`/api`) and the web
|
||||
interface, from voyager 1 (the current build), with categories from the
|
||||
name; voyager 1's 29 learned garbage signatures (`rules:signatures`,
|
||||
`database/data/voyager1_signatures.tsv`).
|
||||
|
||||
Deployment (2026-10-10): `Dockerfile` (targets app and edge),
|
||||
`deploy/compose.yaml` (project usenet-rebuild on homelab), `deploy/deploy.sh`;
|
||||
a push to `rebuild` that passes CI is deployed. See `deploy/README.md`.
|
||||
|
||||
Not built yet: re-running ranges for a retracted rule (the counts and rule
|
||||
versions are recorded), decision f, phase
|
||||
2 (inspection of the probe queue: yEnc names, encryption, archives, par2;
|
||||
passwords; spots).
|
||||
|
||||
## Target
|
||||
|
||||
7,000 days (all of Eweka's retention, back to 2008-08) of the 1000 largest
|
||||
`alt.binaries.*` groups. Estimated: 528 B overview lines; 435 B in their
|
||||
main group; 140 B kept segments in ~1.25 B files and ~115 M releases after
|
||||
dropping garbage; 28 TB of compressed overview traffic. Throughput goal:
|
||||
≥ 25 B overview lines/day (~50 connections), so ~3 weeks.
|
||||
|
||||
## Parts
|
||||
|
||||
1. **Coordinator** (server). Plans each group's history as ranges of 2 M
|
||||
articles, leases them to workers, tracks progress and retries. The
|
||||
group list (owner rule) and the garbage signature table go with every
|
||||
lease. Fetches XOVER for the workers (relay: workers have no NNTP
|
||||
account; the gzip stream is passed on as is).
|
||||
|
||||
**Prefetch**: the coordinator fetches the next ranges ahead of the
|
||||
workers, in chunks of 100 k lines and in order, into a spool on disk
|
||||
(~106 MB per 2 M range at ~53 B per line; on the HDD, written and read
|
||||
sequentially). It keeps about two ranges ahead per active lease, capped
|
||||
by size (default 10 GB), and fetches with all its header connections all
|
||||
the time, independent of how fast workers are. A lease goes to a range
|
||||
that is already (partly) spooled, else to the next planned one (leased
|
||||
ranges are fetched first); a worker reads its chunks from the spool,
|
||||
waiting only for chunks still being fetched. A worker gets its
|
||||
whole range in one streaming request (`GET /ranges/:id/stream`): the
|
||||
spooled chunks in order, waiting for those still being fetched. It still
|
||||
parses chunk by chunk (2 M lines are ~560 MB of text) and posts results
|
||||
per batch, moving the range's resume point. Chunks matter on the NNTP
|
||||
side: a compressed XOVER reply can't be resumed, old ranges answer
|
||||
slowly, and chunks of one range can be fetched in parallel. Spooled chunks are
|
||||
deleted when their range's results are stored, and fetched again if a
|
||||
range is leased again later.
|
||||
Order: oldest era first (2008–2018 holds 63% of the readable files on
|
||||
17% of the lines), then forward.
|
||||
One coordinator process handles ~60 MB/s of compressed XOVER (its event
|
||||
loop was 16.5% busy at ~10 MB/s); beyond that, prefetching is split over
|
||||
processes with `coordinator:run --shard=i/n` (ranges with id % n = i − 1,
|
||||
each with its own NNTP_CONNECTIONS; only shard 1 plans, merges and
|
||||
settles).
|
||||
|
||||
2. **Edge worker** (node, TypeScript reusing `lib/`). Measured (`edge:bench`,
|
||||
2026-10-09, after two rounds of work): ~200–215 k headers/s per core on
|
||||
recent a.b.boneless, ~320–340 k on a 2013 a.b.ath range, ~0.7 GB peak
|
||||
per process. **Rust** (`edge/`, the same RangeIndexer path
|
||||
over the same captures; `check.ts` finds every output identical, the
|
||||
decoded payloads included): 3.5–3.8× less CPU (boneless 2.9 vs 11.1
|
||||
CPU-s per 2 M headers, ath 2.15 vs 7.5) and a third of the memory
|
||||
(0.20–0.24 GB). Its gains: no GC, hand-written matching instead of the
|
||||
regexes, mimalloc, libdeflate (half of zlib's inflate time) and SIMD line
|
||||
scanning. At the full target (~1.25 M/s) that is ~1.5–2 worker cores
|
||||
instead of ~6–7; the speed is still set by the NNTP connections. The
|
||||
cost: a second implementation of lib/headers, lib/release, lib/rules and
|
||||
the chunk format, kept identical (check.ts) for as long as both exist.
|
||||
Findings, how to run both and how to compare their outputs:
|
||||
`edge/README.md`. Since 2026-10-09 the Rust worker
|
||||
(`edge-node`, same crate) is the one running on homelab: ~8× less CPU
|
||||
per header and 20× less memory than `edge:node` in production.
|
||||
Per lease: compressed chunks of 10–50 k lines, in order →
|
||||
parse subjects → skip crossposts outside their owner group (of the
|
||||
indexed groups in the article's Xref, the one with the lowest priority
|
||||
number; see the group list) → group articles into files → files
|
||||
into releases (`lib/release` rules) → drop garbage (random name +
|
||||
dropped signature or tool), bare executables, dropped posters, and the
|
||||
single-archive fakes (stub archives and nonsense pieces, below) → set
|
||||
aside per-article random subjects and
|
||||
obfuscated subjects (counted only; phase 2). Returns per batch: release
|
||||
chunks, edge fragments, probe candidates, counters. Holds no state beyond
|
||||
its lease.
|
||||
|
||||
**Probe candidates**: per release, the articles phase 2 should read, as
|
||||
(release key, kind, message-id, file in its chunk): the first article of
|
||||
the first archive volume (`.part01.rar`, `.rar`, `.7z.001`), of the par2
|
||||
index (smallest `.par2`), and of a lone random-named file; for a 7z set
|
||||
also its last volume (the listing is in the tail). At most a few per
|
||||
release (~115 M releases → ~150–250 M articles), not one per file. Per-
|
||||
article and obfuscated subjects get none (they need every article's body).
|
||||
|
||||
3. **Chunk store** (bulk, append-only). One record per release per batch:
|
||||
its files' names, sizes and segment lists, compactly encoded (template
|
||||
codec for message-ids, zstd). Written once, never updated. A batch is
|
||||
stored in one transaction and only when it starts at its range's resume
|
||||
point, so a batch sent twice or late is refused and the catalog's sums
|
||||
stay right (no per-file dedup); a recovery run for a retracted rule only
|
||||
adds what that rule had dropped. ~3 TB.
|
||||
|
||||
4. **Release catalog** (hot, Postgres on NVMe). One row per release: key,
|
||||
name, poster, groups, posted_at, file count, size, segments, expected
|
||||
file count (`[i/N]`), has-par2, readable-name flag, state. Updated with
|
||||
aggregates once per batch (one upsert per release per batch), plus a
|
||||
small name index for search. ~115 M rows, ~50–100 GB.
|
||||
|
||||
5. **Rules and recovery**. What a worker skips isn't stored (message-ids
|
||||
are the bulk: garbage alone would be ~2.5 TB); a skipped range is read
|
||||
again instead (headers are cheap: ~53 B per overview line compressed).
|
||||
- *Per range, per rule and signature*: how many articles and files the
|
||||
rule dropped (garbage per (tool, group), per-article random subjects,
|
||||
obfuscated subjects, and crossposts per owner group). A few rows per
|
||||
range.
|
||||
- *Rule versions*: every range records the version of the rule set it ran
|
||||
with. Retracting or changing a rule makes the ranges where it matched
|
||||
stale; the coordinator leases them again for that rule only, and the
|
||||
worker returns just what the rule had dropped.
|
||||
- *Sample*: ~1% of what each rule drops (by a hash of the release key) is
|
||||
kept with full file details and probe candidates, so a rule can be
|
||||
checked, and its losses measured in phase 2, before it is retracted.
|
||||
|
||||
6. **Probe queue**. The candidates, in the catalog's database: one small row
|
||||
each (release, kind, message-id, chunk reference), priority by the
|
||||
release's state (settled first) and kind (random names before readable
|
||||
ones). Phase 2 takes them from here; phase 1 only fills it.
|
||||
|
||||
7. **Fragments**. Files crossing a range edge: kept by range and article
|
||||
number; merged when the neighbouring range is done, then written as a
|
||||
chunk. Never staged per segment.
|
||||
|
||||
8. **Settling**. A release is settled when nothing was added to it for 6
|
||||
hours (30 minutes once its last post is that old: history), and its
|
||||
completeness decided: complete when every file its [i/N] counter
|
||||
announced was found. Without a counter, by its par2 sets once their index
|
||||
is probed (`par2_completeness`): every file a set describes is there by
|
||||
name or by its first 16 KiB, or, renamed, as many files and bytes; decided
|
||||
again after a par2 merge and before the pruner drops a set's file list.
|
||||
Segments aren't compared. Phase 2 picks its work from settled releases.
|
||||
|
||||
9. **NZB and API**. NZB built from a release's chunks, with a known
|
||||
password in its meta; Newznab search (names, inner names) and get; a web
|
||||
interface. Categories (Newznab's) by a trigger on releases (migration
|
||||
5200): from the name, else the inner name the prober found, else a spot's
|
||||
category (spot_category, set by the spot inspector), else the groups
|
||||
(erotica, sounds, ebooks, …). `releases:recategorize` recomputes old rows
|
||||
in batches.
|
||||
|
||||
## Group list
|
||||
|
||||
Editable at any time: an added group gets its ranges planned, a removed one
|
||||
no new leases. Each group gets a priority number when it is added (the first
|
||||
list by size, later additions after it); a crosspost belongs to the indexed
|
||||
group in its Xref with the lowest number. So adding a group never takes
|
||||
articles from groups already done, and removing one re-reads only the ranges
|
||||
of other groups whose counts show crossposts skipped for it (part 5). Start:
|
||||
the 1000 largest `alt.binaries.*` groups by LIST ACTIVE (~266 k ranges of
|
||||
2 M articles; boneless alone 80 k).
|
||||
|
||||
## Corpus
|
||||
|
||||
The grouping sweeps' dataset (2026-10-12; LOG.md 2026-10-12 (12)): the XOVER
|
||||
chunks of ranges marked `capture`, kept on the HDD in
|
||||
`corpus/<group>/<start>-<end>/` (edge-bench's layout, `range.json` once
|
||||
complete), up to 300 GB. The prefetcher stores them as it fetches and reads
|
||||
them from there afterwards, so a reset re-indexes them without the news
|
||||
server. `deploy/corpus_sweep.py` marks more ranges each sweep (`capture`,
|
||||
spread over groups and eras, the top-100 groups first), replays every stored
|
||||
range with the deployed build's edge-bench (`replay`, ~seconds a range) and
|
||||
reports suspects and the change since the previous build (`report`). A rule
|
||||
change is judged on the whole corpus before a reset.
|
||||
|
||||
Since 2026-10-23 also history fetched ahead of the indexing: while the
|
||||
catalog is over its NVMe cap, ranges marked `capture` (priority >= 0, not
|
||||
leased) are fetched into the corpus by idle loops, at most perGroup /
|
||||
perRange of a group at a time, to be indexed from disk once there's room
|
||||
(LOG.md 2026-10-23 (100)). Stored filtered (`CORPUS_FILTER`, default on):
|
||||
without the lines the edge worker only counts, decided per line as it does
|
||||
(a crosspost another indexed group owns, a subject that names no file part;
|
||||
~48% of headers), the chunk compressed again as zlib; beside it a `.drops`
|
||||
(gzip JSON: rule version, the owners' priorities, per rule the article
|
||||
numbers as differences, crossposts by owner) and `filtered` in range.json.
|
||||
The worker makes the same releases of a filtered chunk; rule_counts lack
|
||||
those two rules' counts. `corpus:filter` filters chunks stored raw. A
|
||||
corpus filtered this way no longer serves a sweep of those two rules.
|
||||
|
||||
Windows by posting time (2026-10-25; voyager 3's test data, LOG.md (104)):
|
||||
`ranges:dates` samples when each active group's articles were posted (one
|
||||
XOVER of 32 articles every 500M articles of the group, 16 … 1000 samples,
|
||||
the median Date of each slice, kept monotone; `group_dates`; over plain
|
||||
connections: Eweka cuts a compressed reply at the first ".\r\n" in its
|
||||
output, which some small replies begin with, `TruncatedReply`, LOG.md
|
||||
2026-10-25 (105)) and estimates
|
||||
every range's posting date from the curve (`ranges.est_date`; the edges of
|
||||
a window are off by up to one sample interval, three weeks in boneless).
|
||||
`ranges:capture --from 2025-01-01 --to 2026-01-01` marks every active
|
||||
group's ranges posted in the window `capture`: planned ones not started at
|
||||
`--priority` (0), done ones as they are, both fetched into the corpus by the
|
||||
fill, which takes capture ranges in priority order, done ones too (they're
|
||||
fetched again, corpus only), and skips those complete (`ranges.stored_at`,
|
||||
set when it sees the range.json). A year of every group is the data to
|
||||
replay voyager 3's 30-day blocks on: marked 2026-10-25, 2016 (priority 0,
|
||||
30B articles, ~0.56 TB) and 2021 (priority 1, 87B, ~1.6 TB); a recent year
|
||||
is ~200B articles, ~3.6 TB.
|
||||
|
||||
## Phase 2
|
||||
|
||||
Built: the prober (2026-10-11; `probe:run`, LOG.md 2026-10-11 (2)). It
|
||||
reads the probe queue's first articles (`lone`, `par2`, `archive`) and
|
||||
stores per release whether it's passworded and its inner name, which search,
|
||||
the web interface and Newznab use.
|
||||
|
||||
Built: the par2 merge (2026-10-12; `releases:merge`, the `merger` service,
|
||||
LOG.md 2026-10-12 (13)). The prober stores per probed file start the MD5 of
|
||||
its first 16 KiB (`file_hashes`) and per par2 body its set's file list
|
||||
(`par2_files`, `par2_sets`), which holds the same hash per file whatever the
|
||||
file was renamed to. Every minute the merger merges the releases of one
|
||||
group, posted within 48 hours of a release holding the set's par2, whose
|
||||
files the set describes, into the one with the most files (chunks,
|
||||
candidates, inspections, counts; the par2's shared name when none was
|
||||
readable); `release_merges` keeps what went where. Obfuscated uploads whose
|
||||
files were each a release of their own (`lone` candidates) become one.
|
||||
Readable files have no hash: a name pass (2026-10-23) joins the releases
|
||||
of the set's poster in its group posted within 12 hours whose files the
|
||||
set all names (no case or punctuation; sidecars, par2 and images aside),
|
||||
as `via = 'name'`: an album's tracks and its index, an index beside its
|
||||
differently spelled video. `releases:merge-names` for sets from before.
|
||||
Also merged: lone files whose inner names share a random stem (`inner`),
|
||||
and small sidecar-only releases into their upload (`sidecar`).
|
||||
It reads what's new an hour (`created_at`) at a time, its place kept after
|
||||
each, and merges only the sets held or matched by two releases or more; a
|
||||
file whose first 16 KiB and size more than 20 files share is a common one and
|
||||
matches nothing (LOG.md 2026-10-20 (85)).
|
||||
A set's file list is dropped once it can't merge anything more: complete (all
|
||||
its files in its holder), judged (every holder junk), or final (every holder
|
||||
inside a run of done ranges, those ranges' candidates probed, its hashes seen
|
||||
by the merger); `par2_sets` stays (LOG.md 2026-10-20 (87)).
|
||||
|
||||
**Backlog: voyager 1's features not in voyager 2** (CONCEPT.md decisions;
|
||||
voyager 1's data exported to `/mnt/bulk/usenet-rebuild/v1-export/` on
|
||||
homelab: `voyager1-tables.dump` with 2.76 M spots, upload signatures,
|
||||
signature probes, dropped releases and groups; `voyager1-release-passwords.csv`
|
||||
with 68,032 releases with a password):
|
||||
|
||||
| Feature | Voyager 1 | In voyager 2 |
|
||||
|---|---|---|
|
||||
| Spotnet spots | D29 | the spotter and the spot inspector (below); new spots only, the export's not imported |
|
||||
| Posted NZBs read into releases (also NZBs inside archives) | D26, D28 | not yet |
|
||||
| Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported |
|
||||
| SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) |
|
||||
| Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only |
|
||||
| 7z listings | D19 | yes (2026-10-23): the prober reads the `7z-tail` (the last segment of the highest volume, or of an unsplit multi-segment .7z) and `lib/sevenzip` lists it from the archive's end header (LZMA-packed, decoded in TypeScript); an encrypted header (`-mhe`) means passworded without names. One release keeps only the tails of its highest volume (`seven_zip_tails`). ZIP: the first segment's local headers (first entries, encryption) |
|
||||
| Names from yEnc bodies for obfuscated subjects | D13, D21, D36 | no: obfuscated and per-article subjects are only counted (part 5); their message-ids aren't stored, so a body path needs those ranges read again |
|
||||
| Garbage dropped after probing (no real name, no password) | D42 (its second filter) | the junk judge hides it (below); nothing dropped yet |
|
||||
|
||||
Live indexing (the tail) runs since 2026-10-09.
|
||||
|
||||
**Junk** (voyager 1's second filter, D42). After the header pass's rules
|
||||
(garbage signatures, executables, per-article and obfuscated subjects), the
|
||||
judge (`releases:judge`, service `judge`) judges releases by what the
|
||||
prober, the merges and the spots found: every 2 minutes the releases whose
|
||||
facts changed (noted by triggers in `judge_dirty`), daily every release (for
|
||||
what time changes: a day after settling, ranges read beside it). In this
|
||||
order (the first rule wins): *dead* (every probed first article gone), *fake* (a film, episode
|
||||
or music name whose content is a program: archive entries all
|
||||
executables, or the file itself one by name or by its first bytes, sniffed
|
||||
`exe`), *repeated* (the same first 16 KiB and size in 20+ releases),
|
||||
*locked* (passworded, no password from a spot, its NZB or the title),
|
||||
*unusable* (random names, no inner name, no par2 or inner-name merge, no
|
||||
spot, a day after settling). Verdicts (`release_verdicts`, with the rule
|
||||
set's version) hide releases from search (`judged` shows them); a verdict
|
||||
whose rule no longer matches is taken back. Nothing is dropped yet.
|
||||
|
||||
**Sidecars** (the merger's third pass). An upload's .nfo, .sfv or .nzb
|
||||
that came apart from it (another name, another counter: "[00/80] … .nzb"
|
||||
beside "[01/94] … .part01.rar") was a release of its own. The settler
|
||||
queues small settled releases (≤ 3 files, < 1 MB) in `fold_candidates`;
|
||||
one whose files are all sidecars joins the poster's bigger release in the
|
||||
same group posted within 10 minutes: the one sharing a word of 4+ letters
|
||||
with its subject, else the one under the same counter total, else none
|
||||
(`release_merges.via` 'sidecar').
|
||||
|
||||
**Spots** (after voyager 1's D29; `lib/spotnet` ported as is). Two
|
||||
processes, one NNTP connection each:
|
||||
|
||||
- **The spotter** (`spots:read`, service `spotter`) reads free.pt's
|
||||
overviews after the last article it saw (`spot_state`; the first time the
|
||||
last `SPOT_BACKFILL`, 20,000, ~3 months), every `SPOT_EVERY_SECONDS`
|
||||
(120). For each spot it reads the full header: the signature is checked
|
||||
there as Spotweb does (Spotnet's keys 2–4; self-signed key 7 by hashcash),
|
||||
and the X-XML gives the description, the website and the NZB's articles.
|
||||
A password the title (`{{pw}}`) or the description names
|
||||
(`passwordFromSpot`) is kept with the spot, with its source. A
|
||||
delete/dispose message marks its target `removed` (a personal one only
|
||||
from the same spotter within 5 days) and takes its links and passwords
|
||||
back from the releases.
|
||||
- **The spot inspector** (`spots:inspect`, service `spot-inspector`) looks
|
||||
inside each listed spot, newest first: it reads the compressed NZB (≤
|
||||
`SPOT_MAX_NZB_SEGMENTS` articles in alt.binaries.ftd), lists its files
|
||||
in `spot_files` (named as the header pass names them, `parseSubject`),
|
||||
and matches each file to the release the header pass built for it: the
|
||||
same release identity (`releaseIdentity` over the file's subject and
|
||||
poster) in one of its groups, the release posted nearest the file within
|
||||
48 h. Matched releases are linked in `release_spots`. Placeholder NZBs
|
||||
(`dummy`), gone articles (`nzb-gone`) and unreadable ones are noted on the
|
||||
spot. Unmatched spots are matched again every hour; after
|
||||
`SPOT_STANDALONE_AFTER_HOURS` (24) a spot becomes a release of its own
|
||||
(source `spot`), whose NZB is the spot's (kept gzipped in `spots.nzb`).
|
||||
- **Passwords are kept apart** in `release_passwords` (release, password,
|
||||
source: `nzb` for the NZB's `<meta type="password">`, else `spot-title` or
|
||||
`spot-text`, and the spot), not on the catalog row; the catalog only says
|
||||
`passworded`.
|
||||
|
||||
## Voyager 3: window indexing (design, 2026-10-25)
|
||||
|
||||
Branch `voyager-3` (a push to it only runs the tests; `rebuild` keeps
|
||||
running on homelab until the reset). Goal: the whole history (~1,220 B
|
||||
articles, 3% indexed on 2026-10-25) indexed and searchable on a 4 TB NVMe
|
||||
and the 15 TB HDD. The catalog costs 3.9 B of NVMe per article today
|
||||
(144 GB for 36.5 B), which puts the full history at ~4.7 TB; the target is
|
||||
≤ 2.5 B per article (~3 TB) with room for the WAL and bloat. Sources:
|
||||
LOG.md 2026-10-24 (102).
|
||||
|
||||
What is wrong today: a release is written as soon as one range sees it,
|
||||
updated by every later range that finds more parts, merged afterwards
|
||||
(fragments, par2, names) and settled by wall-clock quiet time, which means
|
||||
little in a backfill. That churn is most of the WAL (0.6 TB a day) and the
|
||||
20 k write IOPS bursts, and leaves bloat. Half the rows are single-article
|
||||
releases, a third of them one poster's image dump (27.7 M rows, LOG.md
|
||||
(102)).
|
||||
|
||||
1. **One cursor a group (amended 2026-10-26; was: blocks).** A group's
|
||||
history is still fetched as ranges of 2 M articles in chunks of 100 k
|
||||
(the spool, the corpus, the prefetcher stay) and leased as ranges, but
|
||||
a group's ranges are indexed in article order by one worker at a time,
|
||||
and the window below is *carried* from range to range: the next range
|
||||
finishes what the previous one left open. The first design leased
|
||||
blocks of 30 days of posting time, ran a group's blocks in parallel and
|
||||
joined their edges afterwards (a block merger). The user asked why
|
||||
(LOG.md 2026-10-25 (110)): downloading stays parallel through the
|
||||
prefetcher and a Rust thread parses faster than a connection delivers,
|
||||
so one worker a group loses nothing but the end of a reset, when the
|
||||
last big group runs alone. The merger, the block cutting and the
|
||||
`blocks` lease go. Costs: one seam a group where the history meets the
|
||||
tail (the releases in flight then end as two incomplete halves; history
|
||||
runs forward in one pass a group, priority only between groups), and
|
||||
the carried state must survive a restart (5 below). Measured on six
|
||||
consecutive ranges of a.b.ath (2016): one range is 15 hours of posting,
|
||||
so without the carry no release would finish inside a range (LOG.md
|
||||
2026-10-26 (111)).
|
||||
**The group lease (2026-10-26, the user's question in LOG.md (117)):**
|
||||
a worker thread leases a *group*, not a range, and asks the coordinator
|
||||
for the group's next range each time it finished one; the coordinator
|
||||
keeps one cursor a group (`group_leases`: the holder, its heartbeat,
|
||||
the next range in article order, the snapshot it stands at). The
|
||||
answer is the range (id, span, resume point), the rules version and the
|
||||
stream URL — nothing about where the chunks are: the stream serves a
|
||||
stored range's chunks from the corpus file and a live one's from the
|
||||
spool, so a worker needs no mount and no corpus layout, and a remote
|
||||
worker is the same binary with another URL. The overhead is a copy
|
||||
through the node API: the corpus is ~30 B an article compressed (2016;
|
||||
16–18 for 2021 and 2025), eight local workers at the bench's rate are
|
||||
~170 MB/s, which the HDD under the corpus bounds either way; a remote
|
||||
worker is bounded by its link (100 Mbit ≈ 400k articles/s). History
|
||||
therefore no longer waits for the corpus fill: a group's stored ranges
|
||||
are read from disk, an unstored one is fetched live and captured into
|
||||
the corpus on the way, and the prefetcher runs ahead of each held
|
||||
group's cursor (plus the capture work left). A group with no planned
|
||||
range left is the tail: the worker keeps the group and asks again
|
||||
every minute, the planner's tail ranges fall into its order. A lease
|
||||
whose holder went quiet for LEASE_TTL_SECONDS is taken over: the new
|
||||
holder downloads the group's snapshot (5) and redoes the ranges since
|
||||
it; the batch store's keys make the redo idempotent. The replay
|
||||
harness (8) stays for offline runs; the driver on homelab goes once the
|
||||
worker runs.
|
||||
|
||||
2. **The window.** The worker reads a block in article order and keeps its
|
||||
open files and releases in memory (counters, names, sizes; the segment
|
||||
lists spooled to a file per block on the worker's disk). The *frontier*
|
||||
is the median Date of the last 1,024 articles with a plausible date
|
||||
(within 7 days of the frontier; 1,024 such dates in a row that agree
|
||||
among themselves move it, the group having paused); Date headers alone
|
||||
aren't trusted (clock skew, bogus years). 1,024, not more: the median
|
||||
trails the newest article by half the ring, a day in a group of 500
|
||||
articles a day. A release is **finished** when the frontier is 24 h past
|
||||
its last part (`WINDOW_MS`): written once, with its completeness decided
|
||||
then (every announced file there and no file short of parts; par2
|
||||
decides later for those without a counter), never updated by the header
|
||||
pass again. Incomplete releases are kept (user, 2026-10-24). A fill
|
||||
posted after the window is a release of its own (the judge's `repeated`
|
||||
rule already knows reposts). At a lease's end (the tail's poll, a worker
|
||||
stopping; amended 2026-10-26, was: the block's end) the releases the
|
||||
window hasn't closed on are written **provisionally** (`settled_at`
|
||||
null, `finished: false`, flagged `edge`): a release being uploaded
|
||||
shows in the catalog, and the carried window finishes the same row
|
||||
later, since the release key is the identity, the group and the first
|
||||
article; the store's finish path adds the files not sent before. No
|
||||
merger: the carry does the joining. The settler goes: `settled_at` is
|
||||
the finish. The **tail** needs no path of its own: its lease ends leave
|
||||
such provisional releases, the next lease's continuation finishes them.
|
||||
One exception (2026-10-27): a provisional release that is complete
|
||||
(every file its counter announced found) and quiet for six hours is
|
||||
settled by the coordinator (`Settler.settleComplete`, every five
|
||||
minutes), as the finish would: no counted file can still come, and the
|
||||
continuation may be hours away or never (the first run's range leases
|
||||
left 109k provisional rows at their blocks' end edges, 606 of them
|
||||
complete, shown as still collecting; the user, release 10697).
|
||||
|
||||
3. **Albums.** Files of one poster in one owner group whose subjects differ
|
||||
only by a number sequence, or share a directory-like prefix, posted
|
||||
within the window, become one release (`source = 'album'`): its name the
|
||||
shared template (or the prefix), its files kept by name so each stays
|
||||
findable; at least two files, else the single stays a release. This is
|
||||
for subjects that name no release (today's `post`/`stem` singles); a
|
||||
poster's ten different cracks in a day stay ten releases. The stake,
|
||||
from voyager 2's final analysis (LOG.md (112)): 77% of its 83.7 M
|
||||
releases had one file, 50% one segment (0.3% of the segments);
|
||||
a.b.pictures.erotica.porn alone was 27.7 M single-file releases of one
|
||||
poster. A counter total shared by many stems of one poster within the
|
||||
window is a batch counter, not a file count (Yenc-PP-A&A's [n/68067]):
|
||||
`file_total` null for them, not 68,067 incomplete singles.
|
||||
**Built 2026-10-26** (edge/src/indexer.rs, LOG.md (113)): a release the
|
||||
window closed on with one file named by its stem or post, the name with
|
||||
a number and no archive, volume, sidecar or random name, waits in the
|
||||
album of its poster and template (the name's digit runs as `#`, the
|
||||
extension apart: `#_#.jpg.data`) posted within WINDOW_MS of the album's
|
||||
first file, at most ALBUM_MAX_FILES (1,000) a album; the album closes a
|
||||
window after that span and is written as one release (`source: album`,
|
||||
no file total) named by the members' shared numbers (`1481439079025_#
|
||||
.jpg.data`), a lone member as the release it was. The batch store writes
|
||||
the members' names as `release_texts` kind 6 (`member`, the first 20).
|
||||
The directory-like prefix variant is not built (no case seen in the
|
||||
corpus samples). The batch counter: a total ≥ 100 under which one poster
|
||||
started 20+ releases within 100 batches gives none of them a file
|
||||
total. On six a.b.ath ranges of 2016: 46 albums of 790 files among
|
||||
2,099 chunks, most of them a subject parser miss (`[n/60] Shogun I
|
||||
(1980) Shogun I (1980).avi.012`: the counter text taken into the name,
|
||||
a volume each) that the album rule glues back together.
|
||||
|
||||
4. **A slimmer catalog.** `releases` loses its text copies: `poster_id` →
|
||||
`posters(id)` (the table gets an id; 33 B → 4), `group_set_id` →
|
||||
`group_sets` (the distinct group combinations: 139,927 in voyager 2's
|
||||
83.7 M releases, LOG.md (112); 85 B jsonb → 4),
|
||||
`release_key` and `identity_key` as 20-byte `bytea` (41 B each → 20),
|
||||
`source` a smallint. ~150 of ~430 B a row; the poster and key indexes
|
||||
shrink with them (~12 GB of 50 today); `releases_category_newest_idx`
|
||||
(5.3 GB, used once) goes, category browsing filters the newest index.
|
||||
Today's 85 GB would be ~55. Cold tables move to the HDD tablespace:
|
||||
`release_inspections` and `probe_candidates` (28 GB, written once, not
|
||||
read by search); `probe_cache` is there already.
|
||||
|
||||
5. **Memory and disk at the edge.** The busiest group posts ~50 M articles
|
||||
a day; at a file per ~50 articles that is ~1 M open files of ~100 B in
|
||||
memory per block, and their segment lists (~25 B each) in the block's
|
||||
spool file, read back when a release is finished. Per worker thread a
|
||||
few hundred MB at worst. The carried window (amended 2026-10-26) has to
|
||||
survive a restart (deploys restart the workers daily): its counters and
|
||||
names are small (kilobytes for most groups, ~100 MB for the busiest),
|
||||
the open files' segment lists are the bulk (~1.25 GB for a day of the
|
||||
busiest group). A snapshot at every 2 M range would be ~600 B/article
|
||||
of writes, so the state is snapshotted at a lease's end and on a timer,
|
||||
to a file on bulk, and a crash redoes the ranges since the last snapshot
|
||||
from the corpus or the prefetch cache: the deterministic keys make the
|
||||
redo idempotent. **The snapshot goes through the API** (2026-10-26):
|
||||
`PUT /node/groups/:id/snapshot` with the serialized window (the
|
||||
frontier, the open files and releases, the open albums, the upload
|
||||
numbers, the batch-counter table; gzip) and the article it stands at
|
||||
(the last batch's `next`), kept by the coordinator on bulk as
|
||||
`<snapshot dir>/<group id>`; `GET` gives it back to the next holder. A
|
||||
worker snapshots when it gives a group back (a stop) and every
|
||||
SNAPSHOT_MINUTES while it holds one, always right after a batch was
|
||||
stored, so the snapshot's article is a resume point. The replay
|
||||
harness (8) keeps a group's window in one process without one.
|
||||
**Built 2026-10-26** (LOG.md (117)): migration 6500 `group_leases`;
|
||||
`app/services/group_leases.ts` (lease by the most urgent planned
|
||||
range within LEASE_MIN_PRIORITY..LEASE_MAX_PRIORITY, the longest free
|
||||
first; takeover after LEASE_TTL_SECONDS, the ranges from the snapshot's
|
||||
article on planned again; next; heartbeat; release; the snapshot
|
||||
streamed to `SNAPSHOT_DIR/<group id>.gz`); the node API's
|
||||
`groups/lease`, `groups/:id/next|heartbeat|release|snapshot`; the
|
||||
range stream serving a stored range's chunks from the corpus; the
|
||||
prefetcher ahead of held groups' cursors, skipping chunks the corpus
|
||||
has; `RangeIndexer::for_group/begin_range/snapshot/restore` (bincode,
|
||||
the files a table, SNAPSHOT_VERSION 1); `edge-node` holding groups by
|
||||
default (`--ranges` keeps the old path). Tests: the lease order,
|
||||
takeover and redo, the endpoints with a corpus-served stream and the
|
||||
snapshot through the API, the snapshot restoring a window between two
|
||||
ranges (the two-range release one, the album joined). Deployed and
|
||||
started 2026-10-26 22:37 (LOG.md (118)). A chunk that doesn't inflate
|
||||
(a live reply cut short; the spool doesn't check) is reported after
|
||||
the batch before it went out and the range asked for again from that
|
||||
chunk: the window stays (the first run lost it, and the open batch).
|
||||
The lease bounds alone don't select a window: the history planner's
|
||||
oldest buckets are priority 0–2 too, so the first run also took 2026
|
||||
history at 0 (25k ranges in 40 groups, fetched live) while 1,739 of
|
||||
the 2016 corpus ranges sat at voyager 2's leftover priorities (done
|
||||
when the window was marked, planned again by the reset): hence
|
||||
`LEASE_CAPTURE_ONLY` (the lease, next and the prefetcher's run-ahead
|
||||
take capture ranges only) and the one-off repair of the priorities.
|
||||
The stream polls the corpus as well as the spool while it waits for a
|
||||
chunk: the corpus fill claims chunks of the same range ahead of the
|
||||
leased run (the prefetcher's claims are per range and chunk), stores
|
||||
them in the corpus only, and the run leaves them there; a stream that
|
||||
looked at the corpus once, seconds before that store, waited its whole
|
||||
hour on the spool and the worker lost the group and its open batch
|
||||
(three groups on the first morning). A failed stream now posts the
|
||||
open batch before the group goes back, like a stop does, so the
|
||||
window's snapshot survives it; only a batch the coordinator refused
|
||||
(lease lost, too large) still loses the window.
|
||||
**Two lanes a group (2026-10-27; migration 6600, LOG.md (119)).** A
|
||||
group's ranges in article order through one window meant a thread
|
||||
that leased a group for its tail range first waded through the
|
||||
group's history (older articles come first), and a held history
|
||||
group's tail waited behind days of its ranges; and the bounds that ran
|
||||
the 2016 window alone (0..0, capture only) shut the tail out entirely.
|
||||
Now `group_leases` has a row per group and *lane*: the `history` lane
|
||||
works the ranges the bounds hand out (the windows, the history), the
|
||||
`tail` lane the tail planner's (priority at or below the tail's,
|
||||
whatever the bounds), each with its own holder, window, snapshot
|
||||
(`<group>-tail.gz`) and takeover redo (the lane's ranges only, and
|
||||
only those the window did after the snapshot: leased, done since it,
|
||||
or read past it). The edge puts `--tail-threads` of its threads on the
|
||||
tail lane (EDGE_TAIL_THREADS; 1 of 2 or more by default, which is
|
||||
what homelab's 16 run with), the rest on history; the prefetcher runs ahead of every held
|
||||
cursor. The seam a group is the same one as before, once: where the
|
||||
history lane reaches the first range the tail lane did, it is caught
|
||||
up. **The lease order** within a lane and priority is a mix (the
|
||||
user, 2026-10-27): one lease starts from the group with the most
|
||||
planned articles, the next from the longest unheld one (never held
|
||||
first), alternating on a sequence (`group_lease_turns`), so a big
|
||||
group snapshotted at a stop doesn't wait behind hundreds of small
|
||||
never-held groups and a quiet group's tail is visited all the same.
|
||||
**`coordinator:status`** (`app/services/status.ts`, `--json`) prints
|
||||
the catalog's room against `CATALOG_MAX_NVME_GB`, the ranges by band
|
||||
(tail, raised, each capture window, the history by thousand), the
|
||||
last hour's indexed and stored articles, the spool, the tail backlog,
|
||||
the held groups with their snapshots' age and size, the snapshot
|
||||
files, and the phase-2 backlogs (provisional releases, probe
|
||||
candidates, the pending ladder, the judge's dirty marks). **End to
|
||||
end** (`tests/e2e`, suite `e2e`, CI builds the binary first): the Rust
|
||||
worker against the real node API, database, spool and corpus — the
|
||||
corpus-fill race, a failed stream and the restart that restores the
|
||||
snapshot, a restart whose snapshot file is gone, a takeover.
|
||||
|
||||
6. **The corpus stays as it is.** Measured on 300 unfiltered chunks
|
||||
(30 M lines, LOG.md (102)): 43% of lines lack a "(k/N)" counter, and of
|
||||
those 73% are random 32-hex tokens, 27% `[PRiVATE]` hashes, ~0.3% posts
|
||||
with a readable name and no counter. The filter (crosspost, counterless)
|
||||
loses nothing voyager 3 would index; albums work from filtered chunks
|
||||
(the image dump's subjects carry "(1/1)"). Indexing without a counter at
|
||||
all (obfuscated uploads, ~31% of all lines) would need bodies, and the
|
||||
raw corpus (2,328 ranges) for a sweep.
|
||||
|
||||
7. **Where it lives.** The window, blocks and albums are in the Rust edge
|
||||
(`edge/`), the batch store, the planner and leases; the TypeScript
|
||||
RangeIndexer keeps the scanner (the corpus filter, tests) and drops its
|
||||
parity with the Rust worker for the window (the `check.ts` comparison
|
||||
covered the range path, which goes). New migrations rebuild the schema
|
||||
from a reset, no in-place conversion of the 81 M rows.
|
||||
|
||||
8. **Test before the reset.** Replay the corpus (the raw ranges and the
|
||||
captures, several B articles) with the new edge into a scratch database
|
||||
on bulk: NVMe bytes per article (the one number that decides whether
|
||||
4 TB is enough), the share of albums and incomplete releases, and a
|
||||
sample of releases against today's. Speed is measured after the reset,
|
||||
on the NVMe.
|
||||
|
||||
9. **The reset.** Tail first, then history by priority, read from the
|
||||
corpus where stored. Survives: `probe_cache` (HDD, by message-id),
|
||||
`spots`, `groups` and their priorities, `nntp_shares`, the corpus.
|
||||
Recomputed: releases, chunks, inspections, candidates, par2 data,
|
||||
verdicts, file hashes. Resets are paused (2026-10-12); this one waits
|
||||
for the user's go-ahead, and for the new NVMe.
|
||||
|
||||
### Schema (built 2026-10-25; migrations 6000 and 6100)
|
||||
|
||||
Migration 6000 *is* the reset: it truncates every table the indexing
|
||||
recomputes and refuses to run on a catalog with releases unless
|
||||
`VOYAGER3_RESET=1`, so merging `voyager-3` into `rebuild` is the go-ahead.
|
||||
Survives: `groups`, `ranges` (planned again from their start, with their
|
||||
priorities, `capture`, `est_date`, `stored_at`, `plain_chunks` and measured
|
||||
`posted_low/high`), the corpus, `probe_cache`, `spots` (unlinked from
|
||||
releases; the inspector matches them again), `nntp_shares`, `nodes`,
|
||||
`users`, the rule sets.
|
||||
|
||||
- `releases`: created anew, 8-byte columns first: `id`, `posted_at`,
|
||||
`last_posted_at`, `first_article`, `last_article`, `segments`, `bytes`,
|
||||
`settled_at` (the finish; null while a tail release is provisional),
|
||||
`inspected_at`, `first_seen_at`; `group_id`, `group_set_id`, `poster_id`,
|
||||
`file_total`, `files`, `chunks`, `category`; `source smallint`
|
||||
(`lib/release/source.ts`: 1 subject, 2 post, 3 stem, 4 spot, 5 nzb, 6
|
||||
album); `readable`, `has_par2`, `complete`, `passworded`, `sidecar_only`,
|
||||
`edge` (open at its block's end); `release_key` and `identity_key` as
|
||||
20-byte `bytea`; `name`, `inner_name`. Gone: `poster`, `groups`,
|
||||
`last_seen_at`. Indexes: pkey, release_key unique, (identity_key,
|
||||
group_id), newest, (poster_id, newest), passworded, `(group_id,
|
||||
identity_key) WHERE edge` for the block merger, `id WHERE settled_at IS
|
||||
NULL` for the provisional ones. No trigram indexes: search matches
|
||||
`release_texts`. The category trigger reads the groups from
|
||||
`group_sets`; the judge's dirty trigger watches `poster_id` instead of
|
||||
`poster`.
|
||||
- `pending_releases` (migrations 6300 and 6400, the HDD): the unnamed
|
||||
releases outside the catalog (decision f): `release_key` (primary),
|
||||
`identity_key`, `group_id`, `name` (the random stem), `poster` text, the
|
||||
posting span, `first_article`, `last_article`, counts, `file_total`,
|
||||
`partial`, `finished`, `tool`, `candidates` jsonb (the probe targets),
|
||||
`inspections` jsonb (what each probe showed), `named`, `status` (0
|
||||
waiting, 1 named and in the catalog, 2 discarded, 3 exhausted, 4 named
|
||||
awaiting the move, 9 taken), `tries`, `created_at`, `probed_at`; partial
|
||||
indexes on the waiting (by `created_at`) and the named (by `probed_at`).
|
||||
- `pending_signatures` (6400): per (tool, group, poster; poster '' for the
|
||||
tool and group together) `probed`, `named`, `dropped`, `updated_at`.
|
||||
- `release_texts(release_id, kind smallint, text varchar(255))` (6200),
|
||||
primary key on all three, one trigram index (fastupdate): every string
|
||||
search matches against, appended by whoever learns one and never an
|
||||
update of the releases row. Kinds (`lib/release/text_kind.ts`): 1 name
|
||||
(the release name, stored once more so one index serves everything),
|
||||
2 file (the prober's names inside an archive), 3 nfo (an NFO's title),
|
||||
4 par2 (a set's name), 5 media (a container's title); at most 20 per
|
||||
release and kind (`textsToStore`). A repeat insert is a no-op; the
|
||||
judge's dirty mark follows an insert or delete, so a rule can react to a
|
||||
name arriving later. Search: the trigram index yields release ids, joined
|
||||
to `releases` by primary key (never an OR across the two tables, which
|
||||
the planner cannot serve from indexes). `inner_name` stays on `releases`
|
||||
as the one display string. Facets to filter on (resolution, codec,
|
||||
language) are not texts: columns, or a `meta jsonb` with a
|
||||
`jsonb_path_ops` index once one is proven useful.
|
||||
- `posters(id serial, poster unique)` with its trigram index;
|
||||
`group_sets(id serial, groups jsonb unique)`, a sorted array of names.
|
||||
- `blocks` (6100): `group_id`, `start_id`, `end_id`, `next_id` (the resume
|
||||
point), `posted_low/high`, `status`, `priority`, `node_id`, `attempts`,
|
||||
`rule_version`, `headers`, `releases`, `leased_at`, `heartbeat_at`,
|
||||
`done_at`, `error`; indexes todo (priority, id) and leased.
|
||||
`ranges.block_id` points at the block; the ranges' own lease columns stay
|
||||
until the leases move (then a migration drops them).
|
||||
- `release_chunks`, `rule_counts`, `rule_samples`: `range_id` → `block_id`.
|
||||
- Gone: `fragments`, `range_blocks`, `release_final()` (final = finished:
|
||||
`settled_at IS NOT NULL`), the settler, the fragment merger,
|
||||
`releases_category_newest_idx`.
|
||||
- `release_inspections` and `probe_candidates` on `CHUNK_TABLESPACE` (the
|
||||
HDD) with their indexes. Note LOG.md 2026-10-19 (81): the judge's fake
|
||||
rule read all of `release_inspections` every round, 17 min on the HDD;
|
||||
the judge must read it by release or keep what it needs on the NVMe.
|
||||
|
||||
### Batch store (built 2026-10-25)
|
||||
|
||||
`app/services/batch_store.ts`: every release chunk of a batch is one
|
||||
release, inserted once. Its keys (`lib/release/keys.ts`): `identity_key`
|
||||
the identity's SHA-1 as 20 bytes; `release_key` the SHA-1 of the identity,
|
||||
the owner group and the release's first article, so it is deterministic
|
||||
(no numbering under a group lock, no catalog lookups before writing) and a
|
||||
repost, starting at another article, is a release of its own. Its poster
|
||||
and group set by id (`app/services/catalog_refs.ts`: insert what's new,
|
||||
read the ids, cache them per process; the tests forget the cache per
|
||||
test), its source as a code, its name into `release_texts` (kind 1). The
|
||||
chunk's own counts go in as the release's; `complete` is decided then from
|
||||
the counter. A finished release's probe candidates are claimable at once
|
||||
(`ready`, `unreadable`); a provisional tail release (`finished: false` on
|
||||
the chunk, `settled_at` null) is finished by a later chunk of the same
|
||||
release key, which adds the files not sent before (sums, one more chunk),
|
||||
sets `settled_at` and makes its candidates claimable; a finished release
|
||||
sent again is left as it is (`ON CONFLICT … WHERE settled_at IS NULL`).
|
||||
Gone: the join of a chunk into an earlier release of its key (the window
|
||||
keeps an upload together; the block merger joins what a block's end cut),
|
||||
the join across groups (FTD's spread uploads: a gap, to be handled by the
|
||||
block merger or the par2 merger across groups), the numbering of release
|
||||
keys, the group lock, `fragments` (ignored when sent; the Rust edge still
|
||||
sends them). The chunk type carries `finished` and `edge` (both optional;
|
||||
the Rust edge doesn't send them yet, so its releases are finished, not
|
||||
edges). Interim until blocks are leased: chunks, rule counts and samples
|
||||
are keyed by `ranges.block_id`, and a range without a block gets a block
|
||||
of its own in the store (`blockOf`).
|
||||
|
||||
Readers moved with it: search joins `posters` and `group_sets`, matches
|
||||
words in `release_texts` (`r.id IN (SELECT release_id … ILIKE)`), searches
|
||||
posters by id, filters `source` by code and returns the source name; the
|
||||
par2 merger, the sidecar folder and the judge's scraped rule use
|
||||
`poster_id` (`release_merges.from_key` holds the key as hex); the spot
|
||||
inspector matches by bytea identity keys and inserts its spot releases by
|
||||
id references with their title in `release_texts`; the spotter and the NZB
|
||||
service compare the source code. Tests insert fixtures through
|
||||
`tests/functional/helpers/releases.ts` (`insertRelease`: text keys, poster
|
||||
and groups by name, source by name). The unit and functional suites pass.
|
||||
|
||||
### The window in the Rust edge (built 2026-10-25)
|
||||
|
||||
`edge/src/indexer.rs`: the scanner and the file aggregator as before; then
|
||||
the window instead of the range path. `Frontier` keeps a ring of the last
|
||||
`FRONTIER_ARTICLES` (1,024) plausible dates (within `PLAUSIBLE_MS`, 7
|
||||
days, of the frontier; 1,024 implausible dates in a row that agree move
|
||||
it), recomputed once per chunk; established after 64 dated articles
|
||||
(`settle` at the block's end for smaller blocks); its first value is the
|
||||
block's start in posting time. Per chunk: completed files, then the files
|
||||
whose last part is before frontier − `WINDOW_MS` (24 h; undated ones idle
|
||||
for `idle` chunks), go into the open release of their key whose posting
|
||||
span is within `UPLOAD_GAP_MS` (3 h) of theirs, else start one (the
|
||||
upload number per key, distinct within a batch); releases whose last part
|
||||
is before frontier − `WINDOW_MS` are judged and written `finished`, as an
|
||||
`edge` when posted within `WINDOW_MS` of the block's start. At the end
|
||||
(`finish`) everything left is written: finished when the window closed on
|
||||
it, else `finished: false, edge: true`. A file short of parts stays in
|
||||
its release and sets `partial` (the store: `complete = false`). A lone
|
||||
article of a multi-part random file is still counted `per-article`, not
|
||||
written; before the block's end a scrambled name counts as random too
|
||||
(`looks_scrambled`: a random subject and poster per article, 160k of
|
||||
alt.binaries.storage's releases; LOG.md (122)).
|
||||
|
||||
**Single-archive rules (built 2026-10-31; LOG.md (123)-(127)).** The judge
|
||||
(`names::judge`, `RuleSet.judge`) gets each file's articles seen and their
|
||||
bytes, and whether the release is *settled* (the window closed on it: a
|
||||
par2 can still join an open one; the start edge doesn't count, since a par2
|
||||
follows its rar and 7% of the catalog's releases are start edges). A settled
|
||||
release of one unsplit zip, rar or 7z and nothing else is dropped as a
|
||||
**stub archive** (`stub-archive`, key: the legs tripped) when posted to
|
||||
2+ groups, under 100 MB (the fakes stop at ~90 MB; hash-named uploads,
|
||||
real albums and passworded videos above it don't), and (A) its articles
|
||||
are ≤ 256 KiB with the release ≥ 1 MB (Power-Post's 250 kB default; a
|
||||
one-article release always has small articles, which exempts e-books and
|
||||
sidecars), or (X) posted to 4+ groups, or (W) by a webmail poster. A
|
||||
ROT13 volume name (".cnegN.rar" is ".partN.rar": a.b.erotica's passworded
|
||||
videos, LOG.md (128)) is a volume, not a single archive. A release
|
||||
touching a music, e-book, comics, tabs or fonts group is exempt: those
|
||||
communities post real single small archives the same way (karaoke zips to
|
||||
a.b.sounds.karaoke + a.b.test with Power-Post's articles; 16 of 32,308
|
||||
known fakes lost, LOG.md (127)). These are the
|
||||
exe-stub and recycled-installer floods under film and software names:
|
||||
55% of the catalog on 2026-10-30, 99.3% of the known fakes, 0 known
|
||||
content in the 217k sample, verified on 1,500 decoded articles. A settled
|
||||
single archive of one article and 100 kB–1 MB under a release-like,
|
||||
unscrambled name without spaces and without an e-book format in it is
|
||||
dropped as a **nonsense piece** (`nonsense-piece`, key: the upload tool):
|
||||
the newznzb.* pieces (scene-styled or random with a "-XX" suffix), 9% of
|
||||
the catalog; under 100 kB are cue/log sidecars and adverts, and the names
|
||||
with spaces are books and albums. Both rules are switched by the
|
||||
rule set (`dropStubArchives`, `dropNonsensePieces`), sampled like every
|
||||
drop, and mirrored in `lib/rules/rule_set.ts` (the TypeScript indexer
|
||||
judges as each batch completes files, so it passes settled without the
|
||||
window's knowledge). Gone: fragments and the cut-off, `uploads_of` (online now),
|
||||
`check.ts` and the fixture comparison (the parity with the TypeScript
|
||||
indexer; CI runs `edge/bench-fixtures.sh` instead), the `incomplete` count
|
||||
(incomplete releases are kept). The wire carries `finished`, `edge` and
|
||||
`partial`; the store reads them (`partial` into `complete`). Memory: the
|
||||
window holds a day of a group's segments in memory (the busiest group's
|
||||
~50 M); the spool file per block of the design is not built yet.
|
||||
|
||||
### The replay harness (built 2026-10-26)
|
||||
|
||||
`node ace replay:corpus --group <name> [--dir CORPUS_DIR] [--from article]
|
||||
[--to article] [--limit ranges] [--batch-chunks 5] [--no-finish]` (commands/
|
||||
replay_corpus.ts; design 8): a group's complete corpus ranges in article
|
||||
order through one window in the Rust edge (`edge-bench --group <dir>
|
||||
--rules rules.json --wire -`: the window carried from range to range, the
|
||||
end finished after the last range; `edge/src/lease.rs` has the lease JSON
|
||||
the worker and the bench share) and the batch store into a scratch
|
||||
catalog, as the worker would store them: each range leased to a node named
|
||||
`replay`, its batches stored at its resume point (pulled from the bench's
|
||||
stdout as the store keeps up), the range finished when its last batch is
|
||||
in. It refuses a database not named `*_replay` or `*_scratch`.
|
||||
The rules are the scratch catalog's current set and group priorities:
|
||||
copy `groups` and `rule_sets` from the catalog for a faithful replay. The
|
||||
report: releases finished, provisional, edges; complete, incomplete,
|
||||
undecided; readable, with par2; files, segments as a share of the
|
||||
headers; bytes per header of the NVMe tables (releases, release_texts,
|
||||
posters, group_sets, rule counts and samples) and of those on
|
||||
CHUNK_TABLESPACE (release_chunks, probe_candidates, release_inspections).
|
||||
The app image carries `edge-bench` for it (Dockerfile); on homelab the
|
||||
scratch catalog is a second database (`usenet_replay`, default tablespace
|
||||
`nvme` like the catalog, the chunk tables on `pg_default`; migrated with
|
||||
the voyager-3 image built from the deploy cache's repo, `groups` and
|
||||
`rule_sets` copied over), the corpus mounted read-only.
|
||||
`deploy/replay-window.sh` runs a window there: each group whose window
|
||||
ranges are all stored is replayed once with the window's bounds as
|
||||
`--from`/`--to` (the corpus holds other windows of the same group), 4
|
||||
groups at a time, logs and `.done` markers in `.cache/replay/`, looping
|
||||
until the window is done (LOG.md 2026-10-26 (111) has the first local
|
||||
numbers, (112) the homelab run). **Deployed 2026-10-26** (LOG.md (116)):
|
||||
voyager 3 runs on homelab without its edge worker (`.cache/no-edge`:
|
||||
deploy.sh and the nightly resume leave it stopped, since range by range
|
||||
it would split releases at every edge with no merger left), and history
|
||||
comes from the corpus through this driver into the catalog itself
|
||||
(`DB=usenet_rebuild OUT=.cache/replay-catalog`, the harness allowed by
|
||||
`REPLAY_INTO_CATALOG=1`), the prober's ladder, the merger and the judge
|
||||
working on it as it fills; the scratch run's numbers (63 groups) were
|
||||
read before (LOG.md (115)). The tail waits for the window worker.
|
||||
|
||||
Next: the replay on homelab over the 2016 window (the NVMe bytes per
|
||||
header for real), the window worker (design 1 and 5: group leases, the
|
||||
stream serving stored chunks, the snapshot through the API; being built,
|
||||
LOG.md (117)), the prober writing `release_texts` kind 2, the spot/NZB
|
||||
match on pending keys (decision f), the tail's provisional writes at a
|
||||
range end (the window's open releases shown while they upload: the files
|
||||
sent provisionally marked, the finish sending the rest). Gone from the list: block
|
||||
leases, the planner and the block merger (design 1, amended).
|
||||
|
||||
## Decisions
|
||||
|
||||
a. Chunk payload: a write-once `bytea` table in Postgres, partitioned by
|
||||
month, on the HDD; files on disk only if writes still limit. **Accepted.**
|
||||
b. A separate database on the same server. **Accepted.**
|
||||
c. Group list: editable, owner by priority number (above). **Accepted.**
|
||||
d. Reposts: separate releases in phase 1, merged in phase 2. **Accepted.**
|
||||
e. Headers through the coordinator's relay, with prefetch (part 1).
|
||||
**Accepted.**
|
||||
f. **Nothing enters the catalog without a usable name** (amended
|
||||
2026-10-26, the user's direction; before: random-named releases keep
|
||||
no chunk but their rows). A release the window closed on whose files
|
||||
all have random names (`isUnreadableName`, `names::is_unreadable_name`:
|
||||
a name read token by token, so separators, spaces and random
|
||||
"extensions" don't hide a scramble; since 2026-11-02, LOG.md (129),
|
||||
when 15% of the catalog's "readable" names were "Zx5AjNaK-mh7CFg6Hf8uu"
|
||||
and "Description - Lt5nHjlo3RiuBb3C") is no release yet: it goes to `pending_releases`
|
||||
(migration 6300, the HDD) with its keys, counts, article span, upload
|
||||
tool and probe targets, no row in `releases`, `release_texts` or
|
||||
`posters`, no chunk. The pending ladder works them **from least to most
|
||||
effort**: (0) no fetch: a signature verdict (tool and group, poster)
|
||||
learned from earlier probes, a spot or NZB that names the message-ids;
|
||||
(1) a par2 main file's first article (it names the files); (2) the first
|
||||
article of the first file (the yEnc name, an archive's listing in its
|
||||
first 16 KiB); (3) a 7z's tail; then nothing more. A probe that yields
|
||||
a usable name moves the release into the catalog, its segments read
|
||||
again from the corpus (the range stored) or the server; one that
|
||||
yields none counts against its signature, and a signature with
|
||||
≥100 probes and ≤0.5% named (voyager 2's garbage learner, LOG.md
|
||||
2026-10-08 (5)) is **discarded** from then on, a sample of it still
|
||||
probed to notice a change. **By class** (2026-11-02, LOG.md (130)): a
|
||||
pending release's class is its tool, group, size bucket, name shape and
|
||||
partial flag (`lib/release/classes.ts`, `names::pending_class`, the SQL
|
||||
`pending_class()`; `pending_releases.klass`), and `pending_classes` keeps
|
||||
each class's climbs and names; the claim takes the classes with the best
|
||||
record first (`classScore`, a prior for the unexplored, at most a quarter
|
||||
of a claim from one class), the cheapest target article first within one
|
||||
(`target_bytes`), and the oldest of the rest; a class with the learner's
|
||||
thresholds is dropped like a signature, and the rule set's
|
||||
`droppedClasses` keep its releases out of the pending store at the edge,
|
||||
the explored sample excepted. A probed file whose first 16 KiB and size
|
||||
match a named catalog release's (`file_hashes`) takes that name without
|
||||
another fetch (`app/services/hash_twins.ts`), on the ladder and for
|
||||
catalog releases without one. The reason, in the 2016 replay: one poster's
|
||||
hash-named pieces ("Bob", 11 aliases, 3.2 M releases in five small
|
||||
groups) were 76% of the rows and three quarters of the NVMe bytes for
|
||||
11% of the segments and nothing anyone can find; the full history has
|
||||
725 M random-named files (LOG.md 2026-10-09). Built 2026-10-26: the
|
||||
edge sends an unnamed release with an empty payload and its tool, the
|
||||
batch store writes `pending_releases` (LOG.md (114)); the ladder in the
|
||||
prober (`app/services/pending_ladder.ts`, LOG.md (115)): a claim takes
|
||||
the oldest waiting releases' next target (par2, archive, lone, 7z-tail
|
||||
in that order), at least PENDING_SHARE (25%) of each claim next to the
|
||||
catalog's own candidates, answered from `probe_cache` when the article
|
||||
was read before; `usableName` (lib/probe/inspect.ts) takes the largest
|
||||
named file of a par2 set or a listing, or a lone file's yEnc name, that
|
||||
is neither packaging nor random; a name sets status 4, no name the next
|
||||
target or status 3 (exhausted); each finished climb counts on
|
||||
`pending_signatures` for (tool, group, poster) and (tool, group), and a
|
||||
signature with PENDING_MIN_PROBES (100) climbs and at most
|
||||
PENDING_MAX_NAMED (0.5%) named is dropped: every five minutes the
|
||||
waiting releases of dropped signatures are discarded (status 2), the
|
||||
rules' `samplePercent` of them left to climb. The namer
|
||||
(`app/services/pending_namer.ts`, in the prober's process): a named
|
||||
release's article span read again from the corpus (the stored range's
|
||||
chunks) or by XOVER on a connection of its own, through the TypeScript
|
||||
indexer with `partialFiles` (a file given up short of parts stays a
|
||||
partial file of its release, as the Rust window writes it; without it
|
||||
the range path's fragment rule left nothing to find: by 2026-10-29
|
||||
about 52k of 210k named releases failed, all partial), written as a release under the pending key with `source`
|
||||
weak-stem, the name as `inner_name`, its chunk and its candidates (the
|
||||
prober answers those from `probe_cache`: inspections, par2 sets, hashes,
|
||||
the password flag). Not built: the spot/NZB match on pending keys (step
|
||||
0 of the ladder beyond signatures), and a signature by poster alone
|
||||
across groups.
|
||||
g. The account's NNTP connections shared by priority (`nntp_shares`): the
|
||||
coordinator (headers) before the prober, each between its minimum and
|
||||
maximum, 5 kept for the spotter, the spot inspector, the plain
|
||||
connection and reconnects. A process grows only into connections the
|
||||
others closed, so the account never goes over its limit. Replaces the
|
||||
fixed split (38 + 8), under which the prober stayed at 8 while history
|
||||
was paused and the coordinator's connections idle (LOG.md 2026-10-18
|
||||
(75)). **Accepted** 2026-10-18.
|
||||
h. Idle NNTP connections closed: one with nothing to do for
|
||||
`NNTP_IDLE_CLOSE_SECONDS` (60) is closed and opened again with work
|
||||
(the coordinator's pool on its next sync, the plain one too; a dry
|
||||
prober lane; the spotter between rounds; the spot inspector with no
|
||||
spots). Eweka dropped the quiet ones and the next command failed
|
||||
("Spot inspection failed: NNTP connection closed"), and idle ones took
|
||||
the account's connections from downloads. **Accepted** 2026-10-22.
|
||||
+1
-7
@@ -1,4 +1,4 @@
|
||||
import { indexEntities } from '@adonisjs/core/generators'
|
||||
import { indexEntities } from '@adonisjs/core'
|
||||
import { defineConfig } from '@adonisjs/core/app'
|
||||
import { generateRegistry } from '@tuyau/core/hooks'
|
||||
|
||||
@@ -91,12 +91,6 @@ export default defineConfig({
|
||||
name: 'functional',
|
||||
timeout: 30000,
|
||||
},
|
||||
// The Rust edge worker against the real node API (tests/e2e): needs edge/target/release/edge-node.
|
||||
{
|
||||
files: ['tests/e2e/**/*.spec.{ts,js}'],
|
||||
name: 'e2e',
|
||||
timeout: 120000,
|
||||
},
|
||||
],
|
||||
forceExit: false,
|
||||
},
|
||||
|
||||
@@ -17,7 +17,7 @@ export default class AccessTokensController {
|
||||
}
|
||||
|
||||
async destroy({ auth }: HttpContext) {
|
||||
const user = auth.use('api').getUserOrFail()
|
||||
const user = auth.getUserOrFail()
|
||||
if (user.currentAccessToken) {
|
||||
await User.accessTokens.delete(user, user.currentAccessToken.identifier)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import env from '#start/env'
|
||||
import NzbService from '#services/nzb_service'
|
||||
import SearchService from '#services/search_service'
|
||||
import SearchService from '#services/SearchService'
|
||||
import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
|
||||
|
||||
const LIMITS = { max: 100, default: 50 }
|
||||
@@ -18,7 +17,7 @@ const SEARCH_FUNCTIONS: Record<string, number | undefined> = {
|
||||
'book': 7000,
|
||||
}
|
||||
|
||||
/** The configured key, if any. */
|
||||
/** The configured key, if any (a Secret from .env, or a plain string set at runtime). */
|
||||
function configuredApiKey(): string | undefined {
|
||||
const value: any = env.get('NEWZNAB_API_KEY')
|
||||
const key = typeof value === 'string' ? value : value?.release()
|
||||
@@ -30,15 +29,15 @@ function int(value: unknown): number | undefined {
|
||||
}
|
||||
|
||||
/**
|
||||
* Newznab API for Prowlarr, Sonarr, Radarr and the like (from voyager 1):
|
||||
* Newznab API (CONCEPT.md D23) for Prowlarr, Sonarr, Radarr and the like:
|
||||
* GET /api?t=caps|search|tvsearch|movie|music|book|get&apikey=…
|
||||
* Releases with readable names (or an inner name from the prober, which then is the title) that aren't known to miss files.
|
||||
*/
|
||||
export default class NewznabController {
|
||||
async handle({ request, response }: HttpContext) {
|
||||
const qs = request.qs()
|
||||
const t = String(qs.t ?? '')
|
||||
const server = this.server(request)
|
||||
|
||||
const xml = (body: string, type = 'application/xml') => {
|
||||
response.header('Content-Type', `${type}; charset=utf-8`)
|
||||
return response.send(body)
|
||||
@@ -46,12 +45,13 @@ export default class NewznabController {
|
||||
|
||||
const apiKey = configuredApiKey()
|
||||
if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
|
||||
|
||||
if (t === 'caps') return xml(capsXml(server, LIMITS))
|
||||
|
||||
if (t === 'get') {
|
||||
const id = int(qs.id)
|
||||
if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
|
||||
const nzb = await NzbService.build(id)
|
||||
const nzb = await SearchService.nzb(id)
|
||||
if (!nzb) return xml(errorXml(300, 'No such item'))
|
||||
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
|
||||
response.header('Content-Disposition', `attachment; filename="${filename}"`)
|
||||
@@ -69,16 +69,19 @@ export default class NewznabController {
|
||||
const limit = Math.min(int(qs.limit) ?? LIMITS.default, LIMITS.max)
|
||||
const offset = int(qs.offset) ?? 0
|
||||
|
||||
// ID-only lookups (tvdbid, imdbid, …) aren't supported: an empty result rather than everything.
|
||||
// ID-only lookups (tvdbid, imdbid, …) are not supported: caps says so, but
|
||||
// answer with an empty result rather than everything.
|
||||
const q = typeof qs.q === 'string' ? qs.q.trim() : ''
|
||||
const idOnly = !q && ['tvdbid', 'tvmazeid', 'rid', 'imdbid', 'tmdbid'].some((p) => qs[p])
|
||||
|
||||
const result = idOnly
|
||||
? null
|
||||
? { total: 0, releases: [] }
|
||||
: await SearchService.search({
|
||||
q,
|
||||
categories,
|
||||
readable: true,
|
||||
notIncomplete: true,
|
||||
hideContainers: true,
|
||||
maxAgeDays: int(qs.maxage),
|
||||
season: int(qs.season),
|
||||
episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
|
||||
@@ -87,30 +90,34 @@ export default class NewznabController {
|
||||
offset,
|
||||
})
|
||||
|
||||
const items = (result?.releases ?? []).map((r) => {
|
||||
const items = result.releases.map((r: any) => {
|
||||
const tv = /(?:^|[._ -])S(\d{1,2})E(\d{1,3})/i.exec(r.name)
|
||||
return {
|
||||
id: r.id,
|
||||
// An obfuscated release by what the prober found inside, without its extension (what clients parse).
|
||||
title: r.readable || !r.innerName ? r.name : r.innerName.replace(/\.[a-z0-9]{2,4}$/i, ''),
|
||||
title: r.name,
|
||||
category: r.category ?? 8010,
|
||||
size: r.size,
|
||||
files: r.fileCount,
|
||||
postedAt: r.postedAt ? new Date(r.postedAt) : null,
|
||||
poster: r.poster,
|
||||
groups: r.groups ?? [],
|
||||
// From the prober (phase 2): encrypted archive entries or headers.
|
||||
password: r.passworded === true,
|
||||
// 1 only when a password is needed that we can't give; a known one is in our NZB.
|
||||
password: r.passwordState === 'missing',
|
||||
description: r.spotDescription ?? undefined,
|
||||
season: tv ? `S${tv[1].padStart(2, '0')}` : undefined,
|
||||
episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
|
||||
}
|
||||
})
|
||||
return xml(resultsXml(server, items, offset, result?.total ?? 0), 'application/rss+xml')
|
||||
return xml(resultsXml(server, items, offset, result.total), 'application/rss+xml')
|
||||
}
|
||||
|
||||
private server(request: HttpContext['request']): NewznabServer {
|
||||
const proto = request.header('x-forwarded-proto') ?? 'http'
|
||||
const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
|
||||
return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey: configuredApiKey() }
|
||||
return {
|
||||
title: 'Usenet Indexer',
|
||||
baseUrl: `${proto}://${host}`,
|
||||
apiKey: configuredApiKey(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,206 +0,0 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import { Readable } from 'node:stream'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import Leases, { LeaseLost } from '#services/leases'
|
||||
import GroupLeases, { LANES } from '#services/group_leases'
|
||||
import BatchStore, { StaleBatch } from '#services/batch_store'
|
||||
import Spool, { type ChunkSpan } from '#services/spool'
|
||||
import Corpus from '#services/corpus'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { KEEPALIVE, encodeFrame, fromWire, type Lane, type WireBatch } from '#lib/edge/index'
|
||||
|
||||
/** While waiting, an empty frame this often: an HTTP client gives up on a quiet body (Node's after 5 minutes). */
|
||||
const KEEPALIVE_MS = 20_000
|
||||
|
||||
/**
|
||||
* The edge workers' API (REBUILD.md, parts 1–2): a lease, the range's
|
||||
* chunks as one stream, results per batch, and the end of the range.
|
||||
*/
|
||||
export default class NodeController {
|
||||
private node(ctx: HttpContext) {
|
||||
return Number(ctx.auth.use('node').getUserOrFail().id)
|
||||
}
|
||||
|
||||
async lease(ctx: HttpContext) {
|
||||
const lease = await Leases.lease(this.node(ctx))
|
||||
if (!lease) return ctx.response.noContent()
|
||||
return lease
|
||||
}
|
||||
|
||||
// Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and takes its
|
||||
// ranges in article order; the window's snapshot goes through here too, so a worker needs no disk of its own.
|
||||
// `?lane=tail` is the group's tail cursor (its own holder, window and snapshot); history by default.
|
||||
|
||||
/** The lane the request names (`?lane=`), or null after a 400 for one that isn't a lane. */
|
||||
private lane(ctx: HttpContext): Lane | null {
|
||||
const lane = ctx.request.qs().lane ?? 'history'
|
||||
if (LANES.includes(lane)) return lane
|
||||
ctx.response.badRequest({ error: `lane: ${LANES.join(' or ')}, not "${lane}"` })
|
||||
return null
|
||||
}
|
||||
|
||||
async leaseGroup(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
const lease = await GroupLeases.lease(this.node(ctx), lane)
|
||||
if (!lease) return ctx.response.noContent()
|
||||
return lease
|
||||
}
|
||||
|
||||
async nextRange(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
try {
|
||||
const next = await GroupLeases.next(Number(ctx.params.id), this.node(ctx), lane)
|
||||
if (!next) return ctx.response.noContent()
|
||||
return next
|
||||
} catch (error) {
|
||||
if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async groupHeartbeat(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.heartbeat(groupId, nodeId, lane))
|
||||
}
|
||||
|
||||
async releaseGroup(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.release(groupId, nodeId, lane))
|
||||
}
|
||||
|
||||
/**
|
||||
* PUT /groups/:id/snapshot?next=N: the window's serialized state standing at article N, streamed to disk as sent
|
||||
* (gzip; the body parser leaves an octet-stream alone, so the request is still readable here).
|
||||
*/
|
||||
async putSnapshot(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
const next = Number(ctx.request.qs().next)
|
||||
if (!Number.isFinite(next) || next <= 0) return ctx.response.badRequest({ error: 'next (the snapshot\'s article) is needed' })
|
||||
const raw = ctx.request.raw()
|
||||
const body = raw !== null ? Readable.from([Buffer.from(raw, 'binary')]) : ctx.request.request
|
||||
return this.lifecycle(ctx, (groupId, nodeId) => GroupLeases.putSnapshot(groupId, nodeId, next, body, lane))
|
||||
}
|
||||
|
||||
async getSnapshot(ctx: HttpContext) {
|
||||
const lane = this.lane(ctx)
|
||||
if (!lane) return
|
||||
try {
|
||||
const snapshot = await GroupLeases.getSnapshot(Number(ctx.params.id), this.node(ctx), lane)
|
||||
if (!snapshot) return ctx.response.notFound({ error: 'no snapshot' })
|
||||
ctx.response.header('Content-Type', 'application/octet-stream')
|
||||
ctx.response.header('Content-Length', String(snapshot.size))
|
||||
ctx.response.header('X-Snapshot-Next', String(snapshot.next))
|
||||
return ctx.response.stream(snapshot.stream)
|
||||
} catch (error) {
|
||||
if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The range's chunks from its resume point: a stored range's from the corpus (a worker needs no mount; REBUILD.md
|
||||
* design 1), else from the spool, waiting for those still being fetched.
|
||||
*/
|
||||
async stream(ctx: HttpContext) {
|
||||
const nodeId = this.node(ctx)
|
||||
const rangeId = Number(ctx.params.id)
|
||||
const range = await this.leased(ctx, rangeId, nodeId)
|
||||
if (!range) return
|
||||
const spans = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }, Number(range.next_id))
|
||||
const group = range.capture ? await db.from('groups').where('id', range.group_id).select('name').first() : null
|
||||
const corpusRange = group ? { group: group.name as string, start: Number(range.start_id), end: Number(range.end_id) } : null
|
||||
// The chunk from the corpus or the spool, whichever has it. Both are polled while waiting: the corpus fill
|
||||
// claims a chunk ahead of the leased run (the prefetcher's claims are per range and chunk), stores it in the
|
||||
// corpus only, and the run then leaves it there — a stream that looked at the corpus once, before that
|
||||
// store, waited the whole hour on the spool (three groups on 2026-10-27, LOG.md (118)).
|
||||
const stored = async (span: ChunkSpan) =>
|
||||
corpusRange && (await Corpus.has(corpusRange, span)) ? Corpus.read(corpusRange, span) : null
|
||||
async function* frames() {
|
||||
for (const span of spans) {
|
||||
const waitUntil = Date.now() + coordinatorConfig.streamChunkWaitMs
|
||||
let quietSince = Date.now()
|
||||
let data = await stored(span)
|
||||
while (!data) {
|
||||
if (await Spool.has(rangeId, span)) {
|
||||
data = await Spool.read(rangeId, span)
|
||||
break
|
||||
}
|
||||
if (Date.now() > waitUntil) throw new Error(`Chunk ${span.from}–${span.to} of range ${rangeId} wasn't spooled in time`)
|
||||
if (Date.now() - quietSince >= KEEPALIVE_MS) {
|
||||
quietSince = Date.now()
|
||||
yield encodeFrame(KEEPALIVE)
|
||||
await Leases.heartbeat(rangeId, nodeId)
|
||||
}
|
||||
await sleep(500)
|
||||
data = await stored(span)
|
||||
}
|
||||
yield encodeFrame({ ...span, data })
|
||||
// Streaming keeps the lease alive.
|
||||
await Leases.heartbeat(rangeId, nodeId)
|
||||
}
|
||||
}
|
||||
ctx.response.header('Content-Type', 'application/octet-stream')
|
||||
ctx.response.stream(Readable.from(frames()))
|
||||
}
|
||||
|
||||
async batch(ctx: HttpContext) {
|
||||
const nodeId = this.node(ctx)
|
||||
const rangeId = Number(ctx.params.id)
|
||||
try {
|
||||
return await BatchStore.store(rangeId, nodeId, fromWire(ctx.request.body() as WireBatch))
|
||||
} catch (error) {
|
||||
if (error instanceof StaleBatch) return ctx.response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async heartbeat(ctx: HttpContext) {
|
||||
return this.lifecycle(ctx, (rangeId, nodeId) => Leases.heartbeat(rangeId, nodeId))
|
||||
}
|
||||
|
||||
async finish(ctx: HttpContext) {
|
||||
return this.lifecycle(ctx, async (rangeId, nodeId) => {
|
||||
const range = await Leases.leased(rangeId, nodeId)
|
||||
await Leases.finish(rangeId, nodeId)
|
||||
await GroupLeases.rangeDone(Number(range.group_id), Number(range.headers), GroupLeases.laneOf(Number(range.priority)))
|
||||
})
|
||||
}
|
||||
|
||||
/** POST /ranges/:id/corrupt { from }: a chunk that doesn't inflate; the range goes back. */
|
||||
async corrupt(ctx: HttpContext) {
|
||||
const from = Number(ctx.request.input('from'))
|
||||
return this.lifecycle(ctx, (rangeId, nodeId) => Leases.corrupt(rangeId, nodeId, from))
|
||||
}
|
||||
|
||||
async release(ctx: HttpContext) {
|
||||
return this.lifecycle(ctx, (rangeId, nodeId) => Leases.release(rangeId, nodeId))
|
||||
}
|
||||
|
||||
private async lifecycle(ctx: HttpContext, action: (rangeId: number, nodeId: number) => Promise<void>) {
|
||||
try {
|
||||
await action(Number(ctx.params.id), this.node(ctx))
|
||||
return { ok: true }
|
||||
} catch (error) {
|
||||
if (error instanceof LeaseLost) return ctx.response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private async leased(ctx: HttpContext, rangeId: number, nodeId: number) {
|
||||
try {
|
||||
return await Leases.leased(rangeId, nodeId)
|
||||
} catch (error) {
|
||||
if (error instanceof LeaseLost) {
|
||||
ctx.response.conflict({ error: error.message })
|
||||
return null
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,6 @@ import type { HttpContext } from '@adonisjs/core/http'
|
||||
|
||||
export default class ProfileController {
|
||||
async show({ auth, serialize }: HttpContext) {
|
||||
return serialize(UserTransformer.transform(auth.use('api').getUserOrFail()))
|
||||
return serialize(UserTransformer.transform(auth.getUserOrFail()))
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,5 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import NzbService from '#services/nzb_service'
|
||||
import SearchService, { type SortOrder } from '#services/search_service'
|
||||
import { expandCategories } from '#lib/newznab/index'
|
||||
import SearchService, { type SortOrder } from '#services/SearchService'
|
||||
|
||||
const flag = (value: unknown) => value === '1' || value === 'true' || value === true
|
||||
|
||||
@@ -11,34 +9,30 @@ function megabytes(value: unknown): number | undefined {
|
||||
return value !== undefined && value !== '' && Number.isFinite(mb) && mb >= 0 ? Math.round(mb * 1024 * 1024) : undefined
|
||||
}
|
||||
|
||||
/** The catalog for the web interface (REBUILD.md, part 9). */
|
||||
export default class ReleasesController {
|
||||
/** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&unlocked=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
|
||||
/** GET /api/v1/releases?q=&readable=1&complete=1&hide_encrypted=1&min_size=&max_size=(MB)&ext=mkv,flac&source=&poster=&sort=&page=&per_page= */
|
||||
async index({ request }: HttpContext) {
|
||||
const qs = request.qs()
|
||||
const categories = String(qs.cat ?? '')
|
||||
.split(',')
|
||||
.map((c) => Number(c.trim()))
|
||||
.filter((c) => Number.isInteger(c) && c > 0)
|
||||
return SearchService.search({
|
||||
q: typeof qs.q === 'string' ? qs.q : undefined,
|
||||
categories: categories.length ? expandCategories(categories) : undefined,
|
||||
readable: flag(qs.readable),
|
||||
complete: flag(qs.complete),
|
||||
unlocked: flag(qs.unlocked),
|
||||
hideEncrypted: flag(qs.hide_encrypted),
|
||||
includePar2Only: flag(qs.par2_only),
|
||||
source: typeof qs.source === 'string' && qs.source ? qs.source : undefined,
|
||||
poster: typeof qs.poster === 'string' && qs.poster ? qs.poster : undefined,
|
||||
minSize: megabytes(qs.min_size),
|
||||
maxSize: megabytes(qs.max_size),
|
||||
extensions: typeof qs.ext === 'string' && qs.ext ? qs.ext.split(/[\s,]+/) : undefined,
|
||||
sort: qs.sort as SortOrder | undefined,
|
||||
page: Number(qs.page) || 1,
|
||||
perPage: Number(qs.per_page) || 50,
|
||||
// The page shows "1,000+"; counting further made filter clicks slow.
|
||||
// The page shows "1,000+"; counting further made filter clicks take seconds.
|
||||
countLimit: 1000,
|
||||
})
|
||||
}
|
||||
|
||||
/** GET /api/v1/releases/:id, with its files */
|
||||
/** GET /api/v1/releases/:id */
|
||||
async show({ params, response }: HttpContext) {
|
||||
const release = await SearchService.release(Number(params.id))
|
||||
return release ?? response.notFound({ error: 'Release not found' })
|
||||
@@ -46,7 +40,7 @@ export default class ReleasesController {
|
||||
|
||||
/** GET /api/v1/releases/:id/nzb */
|
||||
async nzb({ params, response }: HttpContext) {
|
||||
const nzb = await NzbService.build(Number(params.id))
|
||||
const nzb = await SearchService.nzb(Number(params.id))
|
||||
if (!nzb) return response.notFound({ error: 'Release not found' })
|
||||
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
|
||||
response.header('Content-Type', 'application/x-nzb')
|
||||
@@ -54,6 +48,16 @@ export default class ReleasesController {
|
||||
return response.send(nzb.xml)
|
||||
}
|
||||
|
||||
/** GET /api/v1/releases/:id/image: the image of the release's spot, read from Usenet. */
|
||||
async image({ params, response }: HttpContext) {
|
||||
const image = await SearchService.spotImage(Number(params.id))
|
||||
if (!image) return response.notFound({ error: 'No image' })
|
||||
response.header('Content-Type', image.type)
|
||||
// A spot's image never changes.
|
||||
response.header('Cache-Control', 'public, max-age=604800, immutable')
|
||||
return response.send(image.data)
|
||||
}
|
||||
|
||||
/** GET /api/v1/stats */
|
||||
async stats() {
|
||||
return SearchService.stats()
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import SeedOrchestrator, { LeaseLost } from '#services/SeedOrchestrator'
|
||||
import SeedFetchPool from '#services/SeedFetchPool'
|
||||
import RemoteContentService, { ArticleRelayOff, JobLost } from '#services/RemoteContentService'
|
||||
import type { SeedBatch } from '#lib/seed/wire'
|
||||
|
||||
/**
|
||||
* The seed API (CONCEPT.md D38). Nodes make outgoing HTTPS requests only:
|
||||
* lease a range, post batches of what they found, then finish or release
|
||||
* the range. Bodies may be gzip-compressed (Content-Encoding: gzip).
|
||||
*/
|
||||
export default class SeedController {
|
||||
/** POST /api/v1/seed/lease → 200 { range } or 204 when there's nothing to do. */
|
||||
async lease({ seedNode, response }: HttpContext) {
|
||||
const range = await SeedOrchestrator.lease(seedNode.id)
|
||||
return range ? { range } : response.noContent()
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/ranges/:id/batches with a SeedBatch. */
|
||||
async batch({ seedNode, params, request, response }: HttpContext) {
|
||||
const batch = request.body() as SeedBatch
|
||||
if (!Number.isSafeInteger(batch?.next) || !Array.isArray(batch.files) || !Array.isArray(batch.fragments)) {
|
||||
return response.badRequest({ error: 'Expected { next, headers, files[], fragments[] }' })
|
||||
}
|
||||
return this.leased(response, async () => {
|
||||
const stored = await SeedOrchestrator.batch(seedNode.id, Number(params.id), {
|
||||
...batch,
|
||||
headers: Number(batch.headers) || 0,
|
||||
obfuscated: Number(batch.obfuscated) || 0,
|
||||
deferred: Number(batch.deferred) || 0,
|
||||
})
|
||||
return { stored }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/v1/seed/ranges/:id/chunk?from=&to= → the overviews of those
|
||||
* articles as a zlib/gzip stream (relay nodes fetch through the indexer).
|
||||
*/
|
||||
async chunk({ seedNode, params, request, response }: HttpContext) {
|
||||
if (SeedFetchPool.size === 0) return response.serviceUnavailable({ error: 'Relay is off (SEED_NNTP_CONNECTIONS=0); fetch directly' })
|
||||
const from = Number(request.input('from'))
|
||||
const to = Number(request.input('to'))
|
||||
if (!Number.isSafeInteger(from) || !Number.isSafeInteger(to)) return response.badRequest({ error: 'Expected ?from=&to=' })
|
||||
return this.leased(response, async () => {
|
||||
try {
|
||||
const raw = await SeedOrchestrator.chunk(seedNode.id, Number(params.id), from, to)
|
||||
response.header('content-type', 'application/octet-stream')
|
||||
return response.send(raw)
|
||||
} catch (error) {
|
||||
if (error instanceof RangeError) return response.badRequest({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/ranges/:id/done */
|
||||
async done({ seedNode, params, response }: HttpContext) {
|
||||
return this.leased(response, async () => {
|
||||
await SeedOrchestrator.finish(seedNode.id, Number(params.id))
|
||||
return { ok: true }
|
||||
})
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/ranges/:id/release { error?, uncounted? } */
|
||||
async release({ seedNode, params, request, response }: HttpContext) {
|
||||
const error = request.input('error')
|
||||
const uncounted = request.input('uncounted') === true
|
||||
return this.leased(response, async () => {
|
||||
await SeedOrchestrator.release(seedNode.id, Number(params.id), typeof error === 'string' ? error : undefined, uncounted)
|
||||
return { ok: true }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/v1/seed/inspect/lease { max } → 200 { jobs } or 204: content
|
||||
* inspection and archive walks for the node (CONCEPT.md D41).
|
||||
*/
|
||||
async inspectLease({ seedNode, request, response }: HttpContext) {
|
||||
const max = Math.min(10, Math.max(1, Number(request.input('max')) || 1))
|
||||
const jobs = await RemoteContentService.lease(seedNode.id, max)
|
||||
return jobs.length > 0 ? { jobs } : response.noContent()
|
||||
}
|
||||
|
||||
/**
|
||||
* POST /api/v1/seed/inspect/done { queue, jobId, token, result } where
|
||||
* result is the probe as a JSON string (the body parser would turn the
|
||||
* empty strings inside it into nulls).
|
||||
*/
|
||||
async inspectDone({ request, response }: HttpContext) {
|
||||
const { queue, jobId, token, result } = request.only(['queue', 'jobId', 'token', 'result'])
|
||||
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string' || typeof result !== 'string') {
|
||||
return response.badRequest({ error: 'Expected { queue, jobId, token, result: JSON string }' })
|
||||
}
|
||||
return this.job(response, () => RemoteContentService.finish(queue, jobId, token, JSON.parse(result)))
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/inspect/failed { queue, jobId, token, error } */
|
||||
async inspectFailed({ request, response }: HttpContext) {
|
||||
const { queue, jobId, token, error } = request.only(['queue', 'jobId', 'token', 'error'])
|
||||
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
|
||||
return response.badRequest({ error: 'Expected { queue, jobId, token, error }' })
|
||||
}
|
||||
return this.job(response, () => RemoteContentService.fail(queue, jobId, token, String(error ?? 'unknown error')))
|
||||
}
|
||||
|
||||
/** POST /api/v1/seed/inspect/release { queue, jobId, token }: a stopping node gives a job back. */
|
||||
async inspectRelease({ request, response }: HttpContext) {
|
||||
const { queue, jobId, token } = request.only(['queue', 'jobId', 'token'])
|
||||
if (!RemoteContentService.isQueue(queue) || typeof jobId !== 'string' || typeof token !== 'string') {
|
||||
return response.badRequest({ error: 'Expected { queue, jobId, token }' })
|
||||
}
|
||||
return this.job(response, () => RemoteContentService.release(queue, jobId, token))
|
||||
}
|
||||
|
||||
/** GET /api/v1/seed/articles?id=<message id> → the raw article body; 410 when the server doesn't have it. */
|
||||
async article({ request, response }: HttpContext) {
|
||||
const id = request.input('id')
|
||||
if (typeof id !== 'string' || id.length === 0 || id.length > 250) return response.badRequest({ error: 'Expected ?id=<message id>' })
|
||||
try {
|
||||
const body = await RemoteContentService.article(id)
|
||||
if (!body) return response.gone({ error: 'No such article' })
|
||||
response.header('content-type', 'application/octet-stream')
|
||||
return response.send(body)
|
||||
} catch (error: any) {
|
||||
if (error instanceof ArticleRelayOff) return response.serviceUnavailable({ error: error.message })
|
||||
return response.badGateway({ error: `NNTP: ${error.message}` })
|
||||
}
|
||||
}
|
||||
|
||||
/** GET /api/v1/seed/status */
|
||||
async status() {
|
||||
return SeedOrchestrator.status()
|
||||
}
|
||||
|
||||
/** 409 when the job's lock is gone (it was given to another node meanwhile). */
|
||||
private async job(response: HttpContext['response'], action: () => Promise<unknown>) {
|
||||
try {
|
||||
await action()
|
||||
return { ok: true }
|
||||
} catch (error) {
|
||||
if (error instanceof JobLost) return response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** 409 when the range was given to another node meanwhile; the node should drop it. */
|
||||
private async leased(response: HttpContext['response'], action: () => Promise<unknown>) {
|
||||
try {
|
||||
return await action()
|
||||
} catch (error) {
|
||||
if (error instanceof LeaseLost) return response.conflict({ error: error.message })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import type { HttpContext } from '@adonisjs/core/http'
|
||||
import type { NextFn } from '@adonisjs/core/types/http'
|
||||
import SeedOrchestrator from '#services/SeedOrchestrator'
|
||||
|
||||
declare module '@adonisjs/core/http' {
|
||||
interface HttpContext {
|
||||
/** The seeding node that made this request (CONCEPT.md D38). */
|
||||
seedNode: { id: number; name: string }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Seed API requests carry a node's bearer token (`node ace seed:token
|
||||
* <name>`); anything else gets 401. The seed API is the only part of the app
|
||||
* reachable from the internet (traefik routes only /api/v1/seed there).
|
||||
*/
|
||||
export default class SeedNodeMiddleware {
|
||||
async handle(ctx: HttpContext, next: NextFn) {
|
||||
const header = ctx.request.header('authorization') ?? ''
|
||||
const token = header.startsWith('Bearer ') ? header.slice(7).trim() : ''
|
||||
const node = token ? await SeedOrchestrator.nodeForToken(token) : null
|
||||
if (!node) return ctx.response.unauthorized({ error: 'Unknown or disabled seed node token' })
|
||||
ctx.seedNode = node
|
||||
return next()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
import { BaseModel as AdonisBaseModel } from '@adonisjs/lucid/orm'
|
||||
|
||||
export default class BaseModel extends AdonisBaseModel {
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
import { DateTime } from 'luxon'
|
||||
import { column } from '@adonisjs/lucid/orm'
|
||||
import BaseModel from '#models/base_model'
|
||||
import { loadedSegments, storedSegments } from '#services/FileStore'
|
||||
import type { Segment } from '#lib/nzb/index'
|
||||
|
||||
export default class File extends BaseModel {
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column({ columnName: 'file_key' })
|
||||
declare fileKey: string | null
|
||||
|
||||
@column()
|
||||
declare filename: string
|
||||
|
||||
@column()
|
||||
declare poster: string
|
||||
|
||||
@column()
|
||||
declare date: number
|
||||
|
||||
@column()
|
||||
declare parts: number
|
||||
|
||||
/** Stored compressed (see FileStore); the model sees the list. */
|
||||
@column({ prepare: (value: Segment[]) => storedSegments(value), consume: (value: Buffer | null) => loadedSegments(value) })
|
||||
declare segments: Segment[]
|
||||
|
||||
@column({ columnName: 'first_message_id' })
|
||||
declare firstMessageId: string | null
|
||||
|
||||
@column()
|
||||
declare groups: string[]
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { DateTime } from 'luxon'
|
||||
import { column } from '@adonisjs/lucid/orm'
|
||||
import BaseModel from '#models/base_model'
|
||||
|
||||
export default class Group extends BaseModel {
|
||||
@column({ isPrimary: true })
|
||||
declare id: number
|
||||
|
||||
@column()
|
||||
declare name: string
|
||||
|
||||
@column()
|
||||
declare active: boolean
|
||||
|
||||
/** Every article up to here was fetched successfully (see FetchRangeService). */
|
||||
@column({ columnName: 'last_indexed_id' })
|
||||
declare lastIndexedId: bigint
|
||||
|
||||
/** Download bodies of obfuscated articles to identify them (bandwidth-heavy). */
|
||||
@column({ columnName: 'fetch_bodies' })
|
||||
declare fetchBodies: boolean
|
||||
|
||||
/** How far the scheduler has queued fetch jobs. */
|
||||
@column({ columnName: 'last_queued_id' })
|
||||
declare lastQueuedId: bigint
|
||||
|
||||
@column.dateTime({ autoCreate: true })
|
||||
declare createdAt: DateTime
|
||||
|
||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||
declare updatedAt: DateTime
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
import { NodeSchema } from '#database/schema'
|
||||
import { type AccessToken, DbAccessTokensProvider } from '@adonisjs/auth/access_tokens'
|
||||
|
||||
/** An edge worker (REBUILD.md, part 2); it authenticates with an access token (the `node` guard). */
|
||||
export default class Node extends NodeSchema {
|
||||
static accessTokens = DbAccessTokensProvider.forModel(Node, { table: 'node_access_tokens' })
|
||||
declare currentAccessToken?: AccessToken
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import FileStore from '#services/FileStore'
|
||||
import IngestService from '#services/IngestService'
|
||||
import type { RarInfo } from '#lib/rar/index'
|
||||
import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
|
||||
|
||||
/** Stored per file and release; a listing of thousands of files is not useful. */
|
||||
const MAX_ENTRIES = 500
|
||||
|
||||
export interface ArchiveResult {
|
||||
info: RarInfo
|
||||
releaseId?: number
|
||||
releaseRenamed?: string
|
||||
grouped?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Records what a RAR volume's headers say: the listing on the file, the
|
||||
* release it belongs to (from the shared directory or continued entry, when
|
||||
* the file has none yet), and for first volumes the contents and password
|
||||
* flag on the release. See CONCEPT.md (D19, D21).
|
||||
*/
|
||||
class ArchiveService {
|
||||
async record(file: any, binary: any, info: RarInfo): Promise<ArchiveResult> {
|
||||
const { next: _next, ...listing } = info
|
||||
const stored = { ...listing, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
|
||||
await FileStore.setDetails(Number(file.id), { archive: stored })
|
||||
|
||||
let releaseId = file.release_id ?? binary?.release_id ?? null
|
||||
let grouped = false
|
||||
const identity = rarReleaseIdentity(info.entries)
|
||||
if (!releaseId && identity) {
|
||||
const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
|
||||
{
|
||||
release: identity,
|
||||
poster: binary?.poster ?? file.poster,
|
||||
weakIdentity: true,
|
||||
groups: binary?.groups ?? file.groups ?? [],
|
||||
postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined,
|
||||
},
|
||||
])
|
||||
releaseId = ids.get(identity.key)!
|
||||
grouped = true
|
||||
if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId })
|
||||
await db.from('files').where('id', file.id).update({ release_id: releaseId })
|
||||
}
|
||||
if (!releaseId) return { info }
|
||||
|
||||
const release = await db.from('releases').where('id', releaseId).first()
|
||||
const update: Record<string, unknown> = {}
|
||||
if (info.firstVolume !== false) {
|
||||
update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted)
|
||||
if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries)
|
||||
} else if (info.headersEncrypted) {
|
||||
update.encrypted = true
|
||||
}
|
||||
const better = releaseNameFromContents(info)
|
||||
// A spot's title is kept (CONCEPT.md D29).
|
||||
const renamed =
|
||||
release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined
|
||||
if (renamed) update.name = renamed
|
||||
if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
|
||||
|
||||
if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`)
|
||||
return { info, releaseId, releaseRenamed: renamed, grouped }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A release name from an archive listing: the top-level directory shared by
|
||||
* all entries when it reads like a release name, otherwise the stem of the
|
||||
* largest entry with a readable name (e.g. an archive holding just
|
||||
* `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped.
|
||||
*/
|
||||
export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string | undefined {
|
||||
// Hidden entries name nothing: one poster adds the same 22 GB ".1917-….mkv" decoy to every upload (LOG.md 2026-10-06 (2)).
|
||||
const listed = info.entries.filter((e) => !isHidden(e.name))
|
||||
const entries = listed.filter((e) => !e.directory)
|
||||
const tops = new Set(listed.map((e) => e.name.split(/[\\/]/)[0]))
|
||||
if (tops.size === 1 && listed.some((e) => /[\\/]/.test(e.name))) {
|
||||
const [top] = tops
|
||||
if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top
|
||||
}
|
||||
const candidates = entries
|
||||
.map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 }))
|
||||
.filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem))
|
||||
.filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem))
|
||||
.sort((a, b) => b.size - a.size)
|
||||
return candidates[0]?.stem
|
||||
}
|
||||
|
||||
/** An entry in a hidden directory or a hidden file itself (`.x/…`, `dir/.x`). */
|
||||
export function isHidden(name: string): boolean {
|
||||
return name.split(/[\\/]/).some((part) => part.startsWith('.'))
|
||||
}
|
||||
|
||||
/**
|
||||
* Releases named after a hidden archive entry before those were skipped:
|
||||
* renamed from their stored listing (no downloads). Used by
|
||||
* `release:rename --hidden`.
|
||||
*/
|
||||
export async function renameHiddenNamed(): Promise<number> {
|
||||
const releases = await db
|
||||
.from('releases')
|
||||
.where('name', 'like', '.%')
|
||||
.whereNot('source', 'spot')
|
||||
.whereNotNull('contents')
|
||||
.select('id', 'name', 'contents')
|
||||
let renamed = 0
|
||||
for (const release of releases) {
|
||||
const entries: any[] = Array.isArray(release.contents) ? release.contents : []
|
||||
const fromHidden = entries.some((e) => isHidden(e.name) && fileStem(String(e.name).split(/[\\/]/).pop() ?? '') === release.name)
|
||||
if (!fromHidden) continue
|
||||
const better = releaseNameFromContents({ entries })
|
||||
if (!better || better === release.name) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
renamed++
|
||||
}
|
||||
return renamed
|
||||
}
|
||||
|
||||
/**
|
||||
* Renames random-named releases from their stored archive listing (no
|
||||
* downloads). Used by `release:rename`.
|
||||
*/
|
||||
export async function renameFromStoredContents(): Promise<number> {
|
||||
const releases = await db
|
||||
.from('releases')
|
||||
.whereNotNull('contents')
|
||||
.whereRaw("jsonb_array_length(contents) > 0")
|
||||
.select('id', 'name', 'source', 'contents')
|
||||
let renamed = 0
|
||||
for (const release of releases) {
|
||||
if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue
|
||||
const better = releaseNameFromContents({ entries: release.contents })
|
||||
if (!better || better === release.name) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
renamed++
|
||||
}
|
||||
return renamed
|
||||
}
|
||||
|
||||
/**
|
||||
* Obfuscated names: one long token of mixed letters and digits, no
|
||||
* separators, and the other generated shapes seen on Usenet (LOG.md
|
||||
* 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
|
||||
*/
|
||||
export function looksRandom(name: string): boolean {
|
||||
// An unknown short extension (e.g. ".bin") says nothing about the name.
|
||||
name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||||
// VIDEO_TS, Sample, CD1: real words, but no name.
|
||||
if (isGenericName(name)) return true
|
||||
if (/^[0-9a-f]{16,}$/i.test(name)) return true
|
||||
// 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
|
||||
if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
|
||||
// UUIDs
|
||||
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
|
||||
// 1650777167 (yEncBin), "83111763 n" (rclone uploads)
|
||||
if (/^\d{6,}( n)?$/.test(name)) return true
|
||||
// prvjr7hapd0i1ny14pz4, and rclone's long base32 names
|
||||
if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
|
||||
// wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
|
||||
if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
|
||||
if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
|
||||
// Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
|
||||
return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
|
||||
}
|
||||
|
||||
export default new ArchiveService()
|
||||
@@ -0,0 +1,160 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import env from '#start/env'
|
||||
import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import { segmentsOf } from '#services/FileStore'
|
||||
import { looksRandom, releaseNameFromContents } from '#services/ArchiveService'
|
||||
import { rarVolume } from '#lib/content/index'
|
||||
import type { RarEntry, RarInfo } from '#lib/rar/index'
|
||||
import { probeWalk, type WalkProbe, type WalkTask } from '#lib/content/probe'
|
||||
|
||||
/** Stored per release, as for a first-volume listing (ArchiveService). */
|
||||
const MAX_ENTRIES = 500
|
||||
|
||||
/** Listed sizes below this share of the release's size: the first segment listed only part of it. */
|
||||
const COVERED = 0.7
|
||||
|
||||
/** Inspection queues a walk this long after the first volume, when the release has had time to complete. */
|
||||
const WALK_DELAY_MS = 30 * 60 * 1000
|
||||
|
||||
export interface WalkOutcome {
|
||||
releaseId: number
|
||||
/** Nothing downloaded: why. */
|
||||
skipped?: 'missing' | 'covered' | 'volumes'
|
||||
entries?: number
|
||||
before?: number
|
||||
complete?: boolean
|
||||
stopped?: string
|
||||
segments?: number
|
||||
renamed?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Complete archive listings by walking RAR headers (CONCEPT.md D40). The
|
||||
* first segment of a first volume lists the entries that start in it: for
|
||||
* a set of many small files (an audiobook's MP3s, a music album) that is
|
||||
* the first one or two. The walk reads one segment per further entry and
|
||||
* volume, at low priority: its own queue, one job at a time per collection
|
||||
* worker, and NNTP connections only when inspection isn't waiting for one.
|
||||
*/
|
||||
class ArchiveWalkService {
|
||||
/** Segments one walk may download (~0.7 MB each). */
|
||||
readonly maxSegments = env.get('ARCHIVE_WALK_MAX_SEGMENTS', 400)
|
||||
|
||||
/** A listing that leaves most of the release unaccounted for. */
|
||||
worthWalking(entries: number, listedBytes: number, releaseSize: number): boolean {
|
||||
if (entries === 0 || entries >= MAX_ENTRIES) return false
|
||||
return listedBytes < COVERED * releaseSize
|
||||
}
|
||||
|
||||
/** After inspection listed a first RAR volume whose listing ends inside the set. */
|
||||
async queueAfterInspection(releaseId: number | null, info: RarInfo) {
|
||||
if (!releaseId || info.format === '7z' || info.firstVolume === false) return
|
||||
if (!info.truncated || info.headersEncrypted || info.entries.length === 0) return
|
||||
await QueueService.archiveWalkQueue.add('walk', { releaseId }, { jobId: `walk-${releaseId}`, delay: WALK_DELAY_MS })
|
||||
}
|
||||
|
||||
/**
|
||||
* The database half before a walk (D41): the release's volumes, or the
|
||||
* outcome when there's nothing to walk.
|
||||
*/
|
||||
async prepare(releaseId: number): Promise<{ task: WalkTask } | { outcome: WalkOutcome }> {
|
||||
const release = await db.from('releases').where('id', releaseId).select('id', 'size', 'contents').first()
|
||||
if (!release) return { outcome: { releaseId, skipped: 'missing' } }
|
||||
const before = Array.isArray(release.contents) ? release.contents : []
|
||||
const listed = before.reduce((sum: number, e: any) => sum + Number(e.size ?? 0), 0)
|
||||
if (!this.worthWalking(before.length, listed, Number(release.size))) {
|
||||
return { outcome: await this.done(releaseId, { releaseId, skipped: 'covered' }) }
|
||||
}
|
||||
const volumes = await this.volumes(releaseId)
|
||||
if (!volumes) return { outcome: await this.done(releaseId, { releaseId, skipped: 'volumes' }) }
|
||||
return {
|
||||
task: {
|
||||
kind: 'walk',
|
||||
releaseId,
|
||||
volumes: volumes.map((file) => segmentsOf(file).map(({ number, id }) => ({ number, id }))),
|
||||
maxSegments: this.maxSegments,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** The database half after a walk: the fuller listing, the password flag and a better name. */
|
||||
async apply(releaseId: number, walked: WalkProbe): Promise<WalkOutcome> {
|
||||
const release = await db.from('releases').where('id', releaseId).select('id', 'name', 'source', 'contents', 'encrypted').first()
|
||||
if (!release) return { releaseId, skipped: 'missing' }
|
||||
const before = Array.isArray(release.contents) ? release.contents : []
|
||||
const outcome: WalkOutcome = {
|
||||
releaseId,
|
||||
before: before.length,
|
||||
entries: walked.entries.length,
|
||||
complete: walked.complete,
|
||||
stopped: walked.stopped,
|
||||
segments: walked.segments,
|
||||
}
|
||||
if (walked.entries.length > before.length) {
|
||||
const contents = walked.entries.slice(0, MAX_ENTRIES)
|
||||
const update: Record<string, unknown> = {
|
||||
contents: JSON.stringify(contents),
|
||||
encrypted: Boolean(release.encrypted) || walked.headersEncrypted || walked.entries.some((e) => e.encrypted),
|
||||
}
|
||||
const better = releaseNameFromContents({ entries: walked.entries as RarEntry[] })
|
||||
// The rule of ArchiveService.record: random and post-subject names give way, a spot's title stays.
|
||||
if (better && better !== release.name && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post')) {
|
||||
update.name = better
|
||||
outcome.renamed = better
|
||||
}
|
||||
await db.from('releases').where('id', releaseId).update(update)
|
||||
}
|
||||
return this.done(releaseId, outcome)
|
||||
}
|
||||
|
||||
/** Walks a release here, on the inspection pool's connections when inspection isn't waiting for one. */
|
||||
async walk(releaseId: number): Promise<WalkOutcome> {
|
||||
const prepared = await this.prepare(releaseId)
|
||||
if ('outcome' in prepared) return prepared.outcome
|
||||
return this.apply(releaseId, await probeWalk(prepared.task, fetchLow))
|
||||
}
|
||||
|
||||
private async done(releaseId: number, outcome: WalkOutcome) {
|
||||
await db.from('releases').where('id', releaseId).update({ contents_walked_at: new Date() })
|
||||
return outcome
|
||||
}
|
||||
|
||||
/**
|
||||
* The release's RAR volumes in order, from volume 1 up to the first gap.
|
||||
* Null without a first volume (e.g. volumes with random names).
|
||||
*/
|
||||
private async volumes(releaseId: number): Promise<any[] | null> {
|
||||
const files = await db.from('files').where('release_id', releaseId).select('id', 'filename', 'real_name', 'segments')
|
||||
const sets = new Map<string, Map<number, any>>()
|
||||
for (const file of files) {
|
||||
const volume = rarVolume(String(file.real_name ?? file.filename))
|
||||
if (!volume || segmentsOf(file).length === 0) continue
|
||||
const set = sets.get(volume.base) ?? new Map()
|
||||
if (!set.has(volume.number)) set.set(volume.number, file)
|
||||
sets.set(volume.base, set)
|
||||
}
|
||||
// A release holds one set, now and then a second (a sample, subs): the biggest.
|
||||
const set = [...sets.values()].sort((a, b) => b.size - a.size)[0]
|
||||
if (!set?.has(1)) return null
|
||||
const ordered: any[] = []
|
||||
for (let n = 1; set.has(n); n++) ordered.push(set.get(n))
|
||||
return ordered
|
||||
}
|
||||
}
|
||||
|
||||
/** One article body on a pooled connection, behind inspection's waiting jobs. */
|
||||
async function fetchLow(messageId: string): Promise<Buffer> {
|
||||
const conn = await NntpService.acquire(true)
|
||||
let failed = false
|
||||
try {
|
||||
return await NntpService.timed(NntpService.fetchBody(conn, messageId), 'BODY')
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
export default new ArchiveWalkService()
|
||||
@@ -0,0 +1,112 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import FetchRangeService from '#services/FetchRangeService'
|
||||
import { articleAtDate } from '#lib/nntp/date_search'
|
||||
|
||||
/** Articles XOVERed to find a posting date near an article number. */
|
||||
const PROBE = 200n
|
||||
|
||||
/**
|
||||
* Header backfill (CONCEPT.md D32). `start` gives a group a cursor from
|
||||
* where live indexing began back to the article posted N days ago; `tick`
|
||||
* (every scheduler run) queues low-priority, header-only fetch ranges from
|
||||
* that cursor downwards, newest first, while few are waiting and the
|
||||
* database stays within its size limits.
|
||||
*/
|
||||
class BackfillService {
|
||||
/** Sets the group's backfill to reach back `days` days. Returns its range. */
|
||||
async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> {
|
||||
const group = await db.from('groups').where('name', groupName).first()
|
||||
if (!group) throw new Error(`Unknown group ${groupName}`)
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`)
|
||||
const dateOf = async (n: bigint) => {
|
||||
const { overviews } = await NntpService.timed<any>(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`)
|
||||
const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
|
||||
const date = header ? Date.parse(header.date) : Number.NaN
|
||||
return Number.isNaN(date) ? null : date
|
||||
}
|
||||
const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf)
|
||||
// Live indexing began a little before the group was added: continue from there.
|
||||
const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf)
|
||||
const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began
|
||||
await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() })
|
||||
return next > until ? { from: until, to: next } : null
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
/** Stops a group's backfill (queued ranges still run). */
|
||||
async stop(groupName: string) {
|
||||
await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null })
|
||||
}
|
||||
|
||||
/** Queues the next backfill ranges. Returns how many. */
|
||||
async tick(): Promise<number> {
|
||||
const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill
|
||||
const groups = await db
|
||||
.from('groups')
|
||||
.where('active', true)
|
||||
.whereNotNull('backfill_until_id')
|
||||
.whereRaw('backfill_next_id > backfill_until_id')
|
||||
.orderBy('id')
|
||||
if (groups.length === 0) return 0
|
||||
|
||||
const room = await this.room(maxDefaultGb, maxHddGb)
|
||||
if (room !== true) {
|
||||
logger.warn(`Backfill paused: ${room}.`)
|
||||
return 0
|
||||
}
|
||||
const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count')
|
||||
let free = maxQueued - Number(count)
|
||||
let queued = 0
|
||||
// Round-robin over the groups, one range each per pass.
|
||||
while (free > 0 && groups.length > 0) {
|
||||
for (const group of [...groups]) {
|
||||
if (free <= 0) break
|
||||
const next = BigInt(group.backfill_next_id)
|
||||
const until = BigInt(group.backfill_until_id)
|
||||
const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n
|
||||
await FetchRangeService.queue(group, start, next, { backfill: true })
|
||||
group.backfill_next_id = (start - 1n).toString()
|
||||
await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id })
|
||||
free--
|
||||
queued++
|
||||
if (start - 1n <= until) {
|
||||
groups.splice(groups.indexOf(group), 1)
|
||||
logger.info(`${group.name}: backfill fully queued.`)
|
||||
}
|
||||
}
|
||||
}
|
||||
return queued
|
||||
}
|
||||
|
||||
/** True when both parts of the database are under their limits, else why not. */
|
||||
/**
|
||||
* Whether the database has room for more history: the default tablespace
|
||||
* (NVMe) and tablespace `hdd` against BACKFILL_MAX_NVME_GB/_HDD_GB. Seed
|
||||
* leases (CONCEPT.md D38) check the same limits.
|
||||
*/
|
||||
async room(
|
||||
maxDefaultGb = indexerConfig.backfill.maxDefaultGb,
|
||||
maxHddGb = indexerConfig.backfill.maxHddGb
|
||||
): Promise<true | string> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')`
|
||||
)
|
||||
const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3
|
||||
if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})`
|
||||
if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})`
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
export default new BackfillService()
|
||||
@@ -0,0 +1,144 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import FileStore from '#services/FileStore'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
|
||||
export interface BinaryFilesResult {
|
||||
/** New files. */
|
||||
saved: number
|
||||
/** Binaries whose file already existed (same first segment or file_key). */
|
||||
existing: number
|
||||
/** Conflicted, gone or already linked: nothing to do. */
|
||||
skipped: number
|
||||
/** Not all parts stored yet (the job is retried for these). */
|
||||
incomplete: number[]
|
||||
inspections: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns completed binaries into files (worker:file), a batch at a time
|
||||
* (LOG.md 2026-10-06 (7)): one query each for the binaries, their parts and
|
||||
* files that exist already, one INSERT for the new files (so the files
|
||||
* statement trigger updates each release once), and one UPDATE and DELETE
|
||||
* for the binaries' links and copied parts. It was one job of ~8 queries per
|
||||
* binary, ~700 a minute.
|
||||
*/
|
||||
class BinaryFileService {
|
||||
async store(binaryIds: number[]): Promise<BinaryFilesResult> {
|
||||
const result: BinaryFilesResult = { saved: 0, existing: 0, skipped: 0, incomplete: [], inspections: 0 }
|
||||
const ids = [...new Set(binaryIds)]
|
||||
const binaries = await db
|
||||
.from('binaries')
|
||||
.whereIn('id', ids)
|
||||
.whereNull('conflicted_at')
|
||||
.whereNull('file_id')
|
||||
.select('id', 'file_key', 'name', 'real_name', 'poster', 'posted_at', 'groups', 'release_id', 'total_parts')
|
||||
result.skipped = ids.length - binaries.length
|
||||
if (binaries.length === 0) return result
|
||||
|
||||
const partsOf = new Map<number, { number: number; id: string; size: number }[]>()
|
||||
const parts = await db
|
||||
.from('parts')
|
||||
.whereIn('binary_id', binaries.map((b) => b.id))
|
||||
.orderBy([{ column: 'binary_id' }, { column: 'number' }])
|
||||
.select('binary_id', 'number', 'message_id', 'bytes')
|
||||
for (const part of parts) {
|
||||
const list = partsOf.get(Number(part.binary_id)) ?? []
|
||||
list.push({ number: Number(part.number), id: part.message_id, size: Number(part.bytes) })
|
||||
partsOf.set(Number(part.binary_id), list)
|
||||
}
|
||||
const complete = binaries.filter((b) => {
|
||||
const ok = (partsOf.get(Number(b.id))?.length ?? 0) >= b.total_parts
|
||||
if (!ok) result.incomplete.push(Number(b.id))
|
||||
return ok
|
||||
})
|
||||
if (complete.length === 0) return result
|
||||
|
||||
// Already known as a file, e.g. added from a posted NZB before its own articles completed (id only: the row holds the segment list).
|
||||
const firsts = complete.map((b) => partsOf.get(Number(b.id))![0].id)
|
||||
const byFirst = new Map<string, number>()
|
||||
for (const row of await db.from('files').whereIn('first_message_id', firsts).select('id', 'first_message_id')) {
|
||||
byFirst.set(row.first_message_id, Number(row.id))
|
||||
}
|
||||
const fileIdOf = new Map<number, number>()
|
||||
const fresh = complete.filter((b) => {
|
||||
const existing = byFirst.get(partsOf.get(Number(b.id))![0].id)
|
||||
if (existing) fileIdOf.set(Number(b.id), existing)
|
||||
return !existing
|
||||
})
|
||||
|
||||
const now = new Date()
|
||||
// file_key is unique: a second completion of the same upload inserts nothing.
|
||||
const inserted = await FileStore.insert(
|
||||
fresh.map((b) => {
|
||||
const segments = partsOf.get(Number(b.id))!
|
||||
return {
|
||||
file_key: b.file_key,
|
||||
filename: b.name,
|
||||
poster: b.poster,
|
||||
date: b.posted_at ? new Date(b.posted_at).getTime() : now.getTime(),
|
||||
parts: segments.length,
|
||||
segments,
|
||||
groups: JSON.stringify(b.groups),
|
||||
release_id: b.release_id,
|
||||
real_name: b.real_name,
|
||||
size: segments.reduce((sum, s) => sum + s.size, 0),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}
|
||||
})
|
||||
)
|
||||
const insertedByKey = new Map(inserted.map((row) => [row.file_key, row.id]))
|
||||
const missing = fresh.filter((b) => !insertedByKey.has(b.file_key)).map((b) => b.file_key)
|
||||
const byKey = new Map<string, number>()
|
||||
if (missing.length > 0) {
|
||||
for (const row of await db.from('files').whereIn('file_key', missing).select('id', 'file_key')) byKey.set(row.file_key, Number(row.id))
|
||||
}
|
||||
for (const b of fresh) {
|
||||
const id = insertedByKey.get(b.file_key) ?? byKey.get(b.file_key)
|
||||
if (id) fileIdOf.set(Number(b.id), id)
|
||||
}
|
||||
const fresher = new Map(fresh.map((b) => [b.file_key, b]))
|
||||
result.saved = inserted.length
|
||||
result.existing = fileIdOf.size - inserted.length
|
||||
|
||||
// Links, locked in file_key order like ingest's upsert, so the two can't deadlock.
|
||||
const linked = complete.filter((b) => fileIdOf.has(Number(b.id))).sort((a, b) => a.file_key.localeCompare(b.file_key))
|
||||
if (linked.length > 0) {
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(
|
||||
`SELECT id FROM binaries WHERE id IN (SELECT unnest(?::bigint[])) ORDER BY file_key FOR UPDATE`,
|
||||
[linked.map((b) => b.id)]
|
||||
)
|
||||
// Files queued for inspection below keep their binary until it has its fingerprint (par2 naming, D16).
|
||||
const queued = new Set(inserted.filter((row) => row.file_key && ContentService.isCandidate(String(fresher.get(row.file_key)?.name ?? ''))).map((row) => row.id))
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET file_id = v.file_id, awaiting_inspection = CASE WHEN v.queued THEN true ELSE b.awaiting_inspection END
|
||||
FROM unnest(?::bigint[], ?::bigint[], ?::boolean[]) AS v(id, file_id, queued) WHERE b.id = v.id`,
|
||||
[linked.map((b) => b.id), linked.map((b) => fileIdOf.get(Number(b.id))!), linked.map((b) => queued.has(fileIdOf.get(Number(b.id))!))]
|
||||
)
|
||||
})
|
||||
// Header-derived parts are in the file's segment list now; body-derived ones (with a CRC) stay for split checks (D20).
|
||||
await db.rawQuery(`DELETE FROM parts WHERE binary_id = ANY(?::bigint[]) AND pcrc32 IS NULL`, [linked.map((b) => b.id)])
|
||||
}
|
||||
|
||||
// As for one file before: an SFV posted earlier may list them (D34), then inspection.
|
||||
const added = inserted.flatMap((row) => {
|
||||
const b = row.file_key ? fresher.get(row.file_key) : undefined
|
||||
return b
|
||||
? [{ id: row.id, name: b.real_name ?? b.name, posted: String(b.name), poster: b.poster, date: b.posted_at ? new Date(b.posted_at).getTime() : null, release_id: b.release_id }]
|
||||
: []
|
||||
})
|
||||
if (added.length > 0) await SfvService.claimAll(added.map(({ posted: _p, ...file }) => file))
|
||||
// By the posted name, as before: inspection skips later volumes and readable media names anyway.
|
||||
const jobs = added
|
||||
.filter((file) => ContentService.isCandidate(file.posted))
|
||||
.map((file) => file.id)
|
||||
if (jobs.length > 0) await InspectionQueue.add(jobs)
|
||||
result.inspections = jobs.length
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
export default new BinaryFileService()
|
||||
@@ -0,0 +1,321 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NntpService from '#services/NntpService'
|
||||
import Par2Service, { type Par2Result } from '#services/Par2Service'
|
||||
import ArchiveService, { looksRandom, type ArchiveResult } from '#services/ArchiveService'
|
||||
import NameService from '#services/NameService'
|
||||
import { decodeSegment } from '#lib/yenc/index'
|
||||
import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
|
||||
import {
|
||||
isArchiveStart,
|
||||
isLaterVolume,
|
||||
NICOVIDEO_FILE,
|
||||
NICOVIDEO_RELEASE,
|
||||
nicovideoReleaseName,
|
||||
type ContentType,
|
||||
type NicovideoInfo,
|
||||
} from '#lib/content/index'
|
||||
import { chunkMd5, probeInspection, type InspectProbe, type InspectTask, type TaskSegment } from '#lib/content/probe'
|
||||
import type { Segment } from '#lib/nzb/index'
|
||||
import { fileStem, releaseIdentity } from '#lib/release/index'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { looksLikeFilename } from '#lib/subject/index'
|
||||
import { sevenZipVolume } from '#lib/sevenzip/index'
|
||||
import SevenZipService from '#services/SevenZipService'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ArchiveWalkService from '#services/ArchiveWalkService'
|
||||
import FileStore, { segmentsOf } from '#services/FileStore'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
|
||||
/**
|
||||
* MD5 of the first 16 KiB of a file, as par2 records it, from the body of
|
||||
* the file's first part. Undefined when that part is shorter than 16 KiB but
|
||||
* the file is not (the chunk would span two parts).
|
||||
*/
|
||||
export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined {
|
||||
return chunkMd5(decodeSegment(body).data, fileSize)
|
||||
}
|
||||
|
||||
export interface ContentResult {
|
||||
par2?: Par2Result
|
||||
archive?: ArchiveResult
|
||||
named: number
|
||||
contentType: ContentType | null
|
||||
nzb?: NzbImportResult
|
||||
/** NZBs read from inside an archive. */
|
||||
archiveNzbs?: NzbImportResult[]
|
||||
/** Files an SFV's listing moved into its release. */
|
||||
sfvMoved?: number
|
||||
/** A niconico capture's release name from its API response (D34). */
|
||||
nicovideo?: string
|
||||
/** The real name from the yEnc header, for a file posted under a random subject (D36). */
|
||||
yencName?: string
|
||||
/** Why the file's release was deleted as garbage after this probe (D42). */
|
||||
garbage?: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes names and groupings from file contents (CONCEPT.md D21). For a
|
||||
* completed file with a random name, one article (the first) is enough:
|
||||
* - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
|
||||
* - par2 files are recognised by content and name every file of their set,
|
||||
* - RAR headers give the release grouping, name and contents.
|
||||
* Inspection is prepare (database) → probe (downloads and parsing,
|
||||
* lib/content/probe; here or on a seeding node, D41) → apply (database).
|
||||
*/
|
||||
class ContentService {
|
||||
/** Random or missing names, and named first RAR volumes (for contents and password flag). */
|
||||
isCandidate(name: string) {
|
||||
// Even under a random stem: the first volume and the index .par2 are inspected instead.
|
||||
if (isLaterVolume(name)) return false
|
||||
if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
|
||||
if (isArchiveStart(name)) return true
|
||||
if (/\.nzb$/i.test(name)) return true
|
||||
// SFVs name the files of their release; niconico captures carry their title (D34).
|
||||
if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true
|
||||
// Every 7z volume: the set is listed when its last volume is in (see inspect).
|
||||
if (sevenZipVolume(name) !== null) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads that randomise the subject per article often keep the real file
|
||||
* name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`;
|
||||
* LOG.md 2026-10-02 (2)). A file with a random name takes that name, and
|
||||
* leaves its weak one-file release for the release its name gives, where
|
||||
* the other files of the upload gather. Returns the name, or undefined.
|
||||
*/
|
||||
private async nameFromYenc(file: any, binary: any, yencName: string): Promise<string | undefined> {
|
||||
const current = String(file.real_name ?? file.filename)
|
||||
if (file.real_name || !looksRandom(fileStem(current))) return undefined
|
||||
if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined
|
||||
|
||||
const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null
|
||||
const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1)
|
||||
let releaseId = file.release_id
|
||||
if (weak) {
|
||||
const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true })
|
||||
if (identity) {
|
||||
const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
|
||||
{ release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true },
|
||||
])
|
||||
releaseId = ids.get(identity.key) ?? releaseId
|
||||
}
|
||||
}
|
||||
await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId })
|
||||
if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId })
|
||||
return yencName
|
||||
}
|
||||
|
||||
/**
|
||||
* Names a niconico capture's release after the video, while it still has
|
||||
* the archiver's `<time>_nicovideo_jp_watch_sm<id>` name; its tags join
|
||||
* the search text.
|
||||
*/
|
||||
private async nameNicovideo(file: any, info: NicovideoInfo, id: string): Promise<string | undefined> {
|
||||
if (!file.release_id) return undefined
|
||||
const release = await db.from('releases').where('id', file.release_id).first()
|
||||
if (!release || !NICOVIDEO_RELEASE.test(release.name)) return undefined
|
||||
const name = nicovideoReleaseName(info, id)
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', release.id)
|
||||
.update({ name, search_files: info.tags.join(' ').slice(0, 4000) || null })
|
||||
return name
|
||||
}
|
||||
|
||||
/**
|
||||
* The database half before a probe (CONCEPT.md D41): what to inspect for
|
||||
* a file, or null when there's nothing to do. A later 7z volume becomes its
|
||||
* archive's first volume, once all volumes are in.
|
||||
*/
|
||||
async prepare(fileId: number): Promise<InspectTask | null> {
|
||||
const file = await db.from('files').where('id', fileId).first()
|
||||
if (!file || !this.isCandidate(file.filename)) return null
|
||||
// Stored before its signature was found to be garbage (D42): not worth an article.
|
||||
if (await GarbageFilter.skipProbe(file)) return null
|
||||
if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
|
||||
const volumes = await SevenZipService.volumes(file)
|
||||
if (!volumes || (await FileStore.details(Number(volumes[0].id))).archive) return null
|
||||
return this.prepare(Number(volumes[0].id))
|
||||
}
|
||||
const segments = segmentsOf(file)
|
||||
if (!segments[0]?.id) return null
|
||||
// A 7z listing reads the last volume's tail (sniffing decides whether it's 7z: random names too).
|
||||
const volumes = await SevenZipService.volumes(file)
|
||||
const last = volumes && volumes.length > 1 ? segmentsOf(volumes[volumes.length - 1]) : segments
|
||||
return {
|
||||
kind: 'inspect',
|
||||
fileId: Number(file.id),
|
||||
name: String(file.real_name ?? file.filename),
|
||||
segments: taskSegments(segments),
|
||||
sevenZip: volumes ? { volumes: volumes.length, last: taskSegments(last, true) } : null,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The database half after a probe: names, groupings, listings and flags from
|
||||
* what it found. Then the release is deleted if it turned out to be garbage (D42).
|
||||
*/
|
||||
async apply(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
|
||||
let result: ContentResult | undefined
|
||||
// Another probe deleted a release this one writes to as garbage meanwhile (LOG.md 2026-10-08 (7)):
|
||||
// the file's own release (nothing left to do), or one its par2 set joins. Again once that
|
||||
// deletion has committed: the set then finds the release's files gone and makes its own.
|
||||
for (const wait of [0, 1000, 3000]) {
|
||||
if (wait) await new Promise((resolve) => setTimeout(resolve, wait))
|
||||
try {
|
||||
result = await this.applyProbe(task, probe)
|
||||
break
|
||||
} catch (error: any) {
|
||||
if (error.code !== '23503' || wait === 3000) throw error
|
||||
if (!(await db.from('files').where('id', task.fileId).select('id').first())) {
|
||||
return { named: 0, contentType: probe.contentType, garbage: 'deleted meanwhile' }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!result) throw new Error(`Applying the probe of file ${task.fileId} gave no result`)
|
||||
result.garbage = (await GarbageFilter.afterProbe(task.fileId, probe.contentType)) ?? undefined
|
||||
return result
|
||||
}
|
||||
|
||||
private async applyProbe(task: InspectTask, probe: InspectProbe): Promise<ContentResult> {
|
||||
const result: ContentResult = { named: 0, contentType: probe.contentType }
|
||||
const file = await db.from('files').where('id', task.fileId).first()
|
||||
if (!file) return result
|
||||
const binary = await db.from('binaries').where('file_id', task.fileId).first()
|
||||
const { contentType } = probe
|
||||
await FileStore.setDetails(Number(file.id), { content_type: contentType })
|
||||
if (contentType === 'missing') {
|
||||
// Nothing to read; the binary needn't wait for inspection any more.
|
||||
if (binary) await db.from('binaries').where('id', binary.id).update({ awaiting_inspection: null })
|
||||
return result
|
||||
}
|
||||
|
||||
// Fingerprint, so par2 sets (now or later) can name this file exactly.
|
||||
if (binary) {
|
||||
await db
|
||||
.from('binaries')
|
||||
.where('id', binary.id)
|
||||
.update({
|
||||
size: db.raw('coalesce(size, ?)', [probe.size]),
|
||||
md5_16k: db.raw('coalesce(md5_16k, ?)', [probe.md5_16k]),
|
||||
// Inspected: the GC may delete it again by its usual rules (LOG.md 2026-10-06 (11)).
|
||||
awaiting_inspection: null,
|
||||
})
|
||||
}
|
||||
|
||||
result.yencName = await this.nameFromYenc(file, binary, probe.yencName ?? '')
|
||||
if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
|
||||
// An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
|
||||
// to read, and useless without the key.
|
||||
const releaseId = file.release_id ?? binary?.release_id
|
||||
if (releaseId) {
|
||||
await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true })
|
||||
}
|
||||
return result
|
||||
}
|
||||
const nicovideo = NICOVIDEO_FILE.exec(task.name)
|
||||
if (nicovideo) {
|
||||
if (probe.nicovideo) result.nicovideo = await this.nameNicovideo(file, probe.nicovideo, nicovideo[1])
|
||||
return result
|
||||
}
|
||||
if (/\.sfv$/i.test(task.name)) {
|
||||
if (typeof probe.text === 'string') result.sfvMoved = (await SfvService.attach(file, probe.text)) ?? undefined
|
||||
return result
|
||||
}
|
||||
if (contentType === 'nzb') {
|
||||
if (typeof probe.text === 'string') result.nzb = (await NzbImportService.import(file, probe.text)) ?? undefined
|
||||
return result
|
||||
}
|
||||
if (probe.par2 && binary) {
|
||||
result.par2 = await Par2Service.attach(binary, probe.par2)
|
||||
} else if (probe.archive && probe.archive.format === '7z') {
|
||||
result.archive = await ArchiveService.record(file, binary, probe.archive)
|
||||
} else if (probe.archive) {
|
||||
result.archive = await ArchiveService.record(file, binary, probe.archive)
|
||||
// The rest of the listing, later and at low priority (D40).
|
||||
await ArchiveWalkService.queueAfterInspection(result.archive.releaseId ?? null, probe.archive)
|
||||
if (probe.archiveNzbs?.length) {
|
||||
result.archiveNzbs = []
|
||||
for (const { entryName, xml } of probe.archiveNzbs) {
|
||||
const imported = await NzbImportService.import(file, xml, { entryName, moveContainer: (probe.archiveNzbEntries ?? 0) === 1 })
|
||||
if (imported) result.archiveNzbs.push(imported)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (binary) result.named = await NameService.resolve([binary.id])
|
||||
return result
|
||||
}
|
||||
|
||||
/** Inspects a file here: prepare, probe on the inspection pool's connections, apply. */
|
||||
async inspect(fileId: number): Promise<ContentResult | null> {
|
||||
const task = await this.prepare(fileId)
|
||||
if (!task) return null
|
||||
const probe = await probeInspection(task, fetchPooled, (message) => logger.warn(message))
|
||||
return this.apply(task, probe)
|
||||
}
|
||||
}
|
||||
|
||||
/** A task's segment list: all of a small file's; of a big one, the first and the last few (a 7z tail). */
|
||||
function taskSegments(segments: Segment[], tailOnly = false): TaskSegment[] {
|
||||
const list = segments.map(({ number, id, size }) => ({ number, id, size }))
|
||||
if (list.length <= TASK_SEGMENTS) return list
|
||||
return tailOnly ? list.slice(-TASK_SEGMENTS) : [list[0], ...list.slice(-TASK_SEGMENTS)]
|
||||
}
|
||||
|
||||
/** More than whole files (40 parts) and par2 files (20) are read; a 7z tail of 16 MB fits easily. */
|
||||
const TASK_SEGMENTS = 40
|
||||
|
||||
/** One article body on a pooled connection. */
|
||||
async function fetchPooled(messageId: string): Promise<Buffer> {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
|
||||
} catch (error: any) {
|
||||
// A missing article (430) leaves the connection fine.
|
||||
failed = !/^4\d\d/.test(String(error.message))
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One line per finding, as worker:collection logs them (and the seed API
|
||||
* for nodes' results).
|
||||
*/
|
||||
export function describeContent(filename: string, content: ContentResult | null): string[] {
|
||||
const lines: string[] = []
|
||||
if (content?.par2) {
|
||||
const { par2 } = content
|
||||
lines.push(`Par2 "${filename}": release "${par2.releaseName}", ${par2.listed} files listed.`)
|
||||
} else if (content?.archive) {
|
||||
const { info, releaseRenamed, grouped } = content.archive
|
||||
const flags = [info.headersEncrypted && 'encrypted headers', info.entries.some((e) => e.encrypted) && 'encrypted files']
|
||||
.filter(Boolean)
|
||||
.join(', ')
|
||||
lines.push(
|
||||
`Archive "${filename}": ${info.format}, ${info.entries.length} entries${flags ? ` (${flags})` : ''}` +
|
||||
(grouped ? ', grouped into a release' : '') +
|
||||
(releaseRenamed ? `, release renamed to "${releaseRenamed}"` : '')
|
||||
)
|
||||
}
|
||||
if (content?.named) lines.push(`"${filename}" got its real name from par2.`)
|
||||
if (content?.contentType === 'missing') lines.push(`"${filename}": the server no longer has its first article.`)
|
||||
if (content?.contentType === 'nzbe') lines.push(`"${filename}" is an encrypted NZB container.`)
|
||||
if (content?.contentType === 'rclone') lines.push(`"${filename}" is rclone-encrypted.`)
|
||||
if (content?.yencName) lines.push(`"${filename}" is "${content.yencName}" (yEnc name).`)
|
||||
if (content?.garbage) lines.push(`"${filename}": release deleted as garbage (${content.garbage === 'deleted meanwhile' ? 'by another probe' : `${content.garbage}, no name or password`}).`)
|
||||
if (content?.nicovideo) lines.push(`"${filename}" → "${content.nicovideo}".`)
|
||||
if (content?.sfvMoved) lines.push(`SFV "${filename}" moved ${content.sfvMoved} files into its release.`)
|
||||
if (content?.contentType === 'yencbin') lines.push(`"${filename}" is a yEncBin encrypted container.`)
|
||||
if (content?.nzb) {
|
||||
const n = content.nzb
|
||||
lines.push(`NZB "${filename}": ${n.listed} files (${n.known} known, ${n.added} added) → "${n.releaseName}".`)
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
export default new ContentService()
|
||||
@@ -0,0 +1,264 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import { type OverviewHeader } from '#services/IngestService'
|
||||
import { decodeSegment, inferTotalParts, parseYencMeta } from '#lib/yenc/index'
|
||||
import { sniffContentType } from '#lib/content/index'
|
||||
import { parseRarHeaders } from '#lib/rar/index'
|
||||
|
||||
/** Expanding re-reads the file's article range with XOVER; beyond this it is too costly. */
|
||||
const EXPAND_MAX_RANGE = 200_000
|
||||
/** A claimed row whose worker died is taken again after this long. */
|
||||
const RECLAIM_AFTER = '1 hour'
|
||||
|
||||
export interface DeferredArticle {
|
||||
articleNumber: number
|
||||
header: OverviewHeader
|
||||
}
|
||||
|
||||
/** File name from a deferred uploader's subject: `[i/M] "name" yEnc`, no part counter. */
|
||||
export function deferredName(subject: string | undefined): string | null {
|
||||
return /"([^"]+)"/.exec(subject ?? '')?.[1]?.trim() || null
|
||||
}
|
||||
|
||||
export function isDeferred(header: OverviewHeader): boolean {
|
||||
const id = header['message-id'] ?? ''
|
||||
return indexerConfig.deferBodyMessageIds.some((re) => re.test(id))
|
||||
}
|
||||
|
||||
/** A recorded range never spans more articles than this (one XOVER to expand it). */
|
||||
const RANGE_MAX_SPAN = 50_000
|
||||
|
||||
export function bodyJob(header: OverviewHeader, group: string, articleNumber: number | bigint) {
|
||||
return {
|
||||
name: 'process-body',
|
||||
data: { header, group, articleNumber: Number(articleNumber) },
|
||||
// By message-id: an article crossposted to several indexed groups is downloaded once.
|
||||
opts: { jobId: `body-${createHash('sha1').update(header['message-id'] ?? '').digest('hex')}` },
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deferred uploads (CONCEPT.md D28). Their articles skip the body queue and
|
||||
* are recorded here per file, from the headers alone. `worker:deferred`
|
||||
* later downloads one body per file for its size, part count and content
|
||||
* type, and sends small files (NZBs in RARs, par2 index files) to the body
|
||||
* queue completely.
|
||||
*/
|
||||
class DeferredService {
|
||||
/**
|
||||
* Records one group's deferred articles: per file when the subject names
|
||||
* the file, otherwise only as article ranges. Returns the number of files.
|
||||
*/
|
||||
async record(groupName: string, articles: DeferredArticle[]): Promise<number> {
|
||||
const named = articles.filter((a) => deferredName(a.header.subject) !== null)
|
||||
await this.recordRanges(groupName, articles.filter((a) => deferredName(a.header.subject) === null))
|
||||
return this.recordFiles(groupName, named)
|
||||
}
|
||||
|
||||
private async recordRanges(groupName: string, articles: DeferredArticle[]) {
|
||||
if (articles.length === 0) return
|
||||
const sorted = [...articles].sort((a, b) => Number(a.articleNumber) - Number(b.articleNumber))
|
||||
const now = new Date()
|
||||
const rows: Record<string, unknown>[] = []
|
||||
for (const article of sorted) {
|
||||
const n = Number(article.articleNumber)
|
||||
const bytes = Number.parseInt(article.header[':bytes'] ?? '0', 10) || 0
|
||||
const last = rows.at(-1)
|
||||
if (last && n - (last.first_article as number) <= RANGE_MAX_SPAN) {
|
||||
last.last_article = n
|
||||
;(last.article_count as number)++
|
||||
last.bytes = (last.bytes as number) + bytes
|
||||
} else {
|
||||
rows.push({ group_name: groupName, first_article: n, last_article: n, article_count: 1, bytes, created_at: now })
|
||||
}
|
||||
}
|
||||
await db.table('deferred_ranges').multiInsert(rows)
|
||||
}
|
||||
|
||||
private async recordFiles(groupName: string, articles: DeferredArticle[]): Promise<number> {
|
||||
const files = new Map<string, {
|
||||
name: string; subject: string; poster: string; first: number; firstId: string; last: number
|
||||
count: number; bytes: number; postedAt: Date | null
|
||||
}>()
|
||||
for (const { articleNumber, header } of articles) {
|
||||
const name = deferredName(header.subject)!
|
||||
const n = Number(articleNumber)
|
||||
const bytes = Number.parseInt(header[':bytes'] ?? '0', 10) || 0
|
||||
const date = header.date ? new Date(header.date) : null
|
||||
const postedAt = date && !Number.isNaN(date.getTime()) ? date : null
|
||||
const file = files.get(name)
|
||||
if (!file) {
|
||||
files.set(name, {
|
||||
name, subject: header.subject ?? '', poster: header.from ?? '', first: n,
|
||||
firstId: (header['message-id'] ?? '').replace(/^<|>$/g, ''), last: n, count: 1, bytes, postedAt,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (n < file.first) {
|
||||
file.first = n
|
||||
file.firstId = (header['message-id'] ?? '').replace(/^<|>$/g, '')
|
||||
}
|
||||
file.last = Math.max(file.last, n)
|
||||
file.count++
|
||||
file.bytes += bytes
|
||||
if (postedAt && (!file.postedAt || postedAt < file.postedAt)) file.postedAt = postedAt
|
||||
}
|
||||
if (files.size === 0) return 0
|
||||
|
||||
const now = new Date()
|
||||
const rows = [...files.values()].map((f) => ({
|
||||
file_key: createHash('sha1').update(`deferred\n${f.name}`).digest('hex'),
|
||||
name: f.name,
|
||||
subject: f.subject,
|
||||
poster: f.poster,
|
||||
group_name: groupName,
|
||||
first_article: f.first,
|
||||
first_message_id: f.firstId,
|
||||
last_article: f.last,
|
||||
article_count: f.count,
|
||||
bytes: f.bytes,
|
||||
posted_at: f.postedAt,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
}))
|
||||
// Counts and the article range only add up within the group the file was first seen in.
|
||||
await db.rawQuery(
|
||||
`INSERT INTO deferred_files AS d (${Object.keys(rows[0]).join(', ')})
|
||||
SELECT * FROM jsonb_to_recordset(?::jsonb) AS r(
|
||||
file_key varchar, name text, subject text, poster text, group_name text, first_article bigint,
|
||||
first_message_id text, last_article bigint, article_count int, bytes bigint, posted_at timestamptz,
|
||||
first_seen_at timestamptz, last_seen_at timestamptz)
|
||||
ON CONFLICT (file_key) DO UPDATE SET
|
||||
last_seen_at = EXCLUDED.last_seen_at,
|
||||
first_message_id = CASE WHEN d.group_name = EXCLUDED.group_name AND EXCLUDED.first_article < d.first_article
|
||||
THEN EXCLUDED.first_message_id ELSE d.first_message_id END,
|
||||
first_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN LEAST(d.first_article, EXCLUDED.first_article) ELSE d.first_article END,
|
||||
last_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN GREATEST(d.last_article, EXCLUDED.last_article) ELSE d.last_article END,
|
||||
article_count = d.article_count + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.article_count ELSE 0 END,
|
||||
bytes = d.bytes + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.bytes ELSE 0 END,
|
||||
posted_at = LEAST(d.posted_at, EXCLUDED.posted_at)`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
return files.size
|
||||
}
|
||||
|
||||
/** Takes up to `limit` files that stopped growing; they are marked `processing`. */
|
||||
async claim(limit: number): Promise<any[]> {
|
||||
const settled = new Date(Date.now() - indexerConfig.deferredSettleMs)
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE deferred_files SET status = 'processing', processed_at = now()
|
||||
WHERE id IN (
|
||||
SELECT id FROM deferred_files
|
||||
WHERE (status = 'pending' AND last_seen_at < ?)
|
||||
OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
|
||||
ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[settled, limit]
|
||||
)
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues the bodies of up to `limit` recorded ranges, oldest first: the
|
||||
* articles are found again with XOVER. Returns ranges and articles queued.
|
||||
*/
|
||||
async expandRanges(limit: number): Promise<{ ranges: number; articles: number }> {
|
||||
const ranges = await db.from('deferred_ranges').whereNull('expanded_at').orderBy('id').limit(limit)
|
||||
let articles = 0
|
||||
for (const range of ranges) {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let overviews: [number, OverviewHeader][]
|
||||
try {
|
||||
await conn.group(range.group_name)
|
||||
overviews = (await conn.xover(String(range.first_article), String(range.last_article))).overviews
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
const jobs = overviews
|
||||
.filter(([, header]) => header && isDeferred(header) && deferredName(header.subject) === null)
|
||||
.map(([articleNumber, header]) => bodyJob(header, range.group_name, articleNumber))
|
||||
for (let i = 0; i < jobs.length; i += 5000) await QueueService.bodyQueue.addBulk(jobs.slice(i, i + 5000))
|
||||
await db.from('deferred_ranges').where('id', range.id).update({ expanded_at: new Date() })
|
||||
articles += jobs.length
|
||||
}
|
||||
return { ranges: ranges.length, articles }
|
||||
}
|
||||
|
||||
/** Reads one body of the file; expands small files into the body queue. Returns the new status. */
|
||||
async process(file: any): Promise<string> {
|
||||
const update: Record<string, unknown> = { processed_at: new Date(), error: null }
|
||||
try {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let body: Buffer
|
||||
try {
|
||||
body = await NntpService.fetchBody(conn, file.first_message_id)
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
|
||||
const meta = parseYencMeta(body)
|
||||
const totalParts = inferTotalParts(meta)
|
||||
Object.assign(update, {
|
||||
yenc_name: meta.begin.name,
|
||||
size: meta.begin.size,
|
||||
total_parts: totalParts,
|
||||
sampled_part: meta.begin.part ?? 1,
|
||||
})
|
||||
if ((meta.begin.part ?? 1) === 1) {
|
||||
const data = decodeSegment(body, { lenientSize: true, verifyCrc: false }).data
|
||||
update.content_type = sniffContentType(data)
|
||||
const rar = parseRarHeaders(data)
|
||||
if (rar) update.encrypted = rar.headersEncrypted || rar.entries.some((e) => e.encrypted)
|
||||
}
|
||||
|
||||
update.status = 'sampled'
|
||||
if (totalParts <= indexerConfig.deferredExpandMaxParts && update.encrypted !== true) {
|
||||
const queued = await this.expand(file)
|
||||
if (queued > 0) update.status = 'expanded'
|
||||
}
|
||||
} catch (error: any) {
|
||||
update.status = 'failed'
|
||||
update.error = String(error.message).slice(0, 500)
|
||||
}
|
||||
await db.from('deferred_files').where('id', file.id).update(update)
|
||||
return update.status as string
|
||||
}
|
||||
|
||||
/** Finds the file's articles again (XOVER over its range) and queues their bodies. */
|
||||
private async expand(file: any): Promise<number> {
|
||||
const first = Number(file.first_article)
|
||||
const last = Number(file.last_article)
|
||||
if (last - first > EXPAND_MAX_RANGE) return 0
|
||||
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let overviews: [number, OverviewHeader][]
|
||||
try {
|
||||
await conn.group(file.group_name)
|
||||
overviews = (await conn.xover(String(first), String(last))).overviews
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
const jobs = overviews
|
||||
.filter(([, header]) => header && deferredName(header.subject) === file.name)
|
||||
.map(([articleNumber, header]) => bodyJob(header, file.group_name, articleNumber))
|
||||
if (jobs.length > 0) await QueueService.bodyQueue.addBulk(jobs)
|
||||
return jobs.length
|
||||
}
|
||||
}
|
||||
|
||||
export default new DeferredService()
|
||||
@@ -0,0 +1,118 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import QueueService from '#services/QueueService'
|
||||
import indexerConfig from '#config/indexer'
|
||||
|
||||
/** A failed range is retried after this long... */
|
||||
const REQUEUE_AFTER_MS = 5 * 60 * 1000
|
||||
/** ...at most this many times; after that it stays visible as a gap. */
|
||||
const MAX_REQUEUES = 5
|
||||
/** A queued range without a job in Redis (e.g. Redis was flushed) is re-queued after this long. */
|
||||
const LOST_AFTER_MS = 60 * 60 * 1000
|
||||
|
||||
interface Group {
|
||||
id: number
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks the article ranges given to worker:fetch. A range only counts as
|
||||
* indexed once the worker reports it done, so failed or lost fetch jobs are
|
||||
* retried instead of silently skipped. See CONCEPT.md (D18).
|
||||
*/
|
||||
class FetchRangeService {
|
||||
/** Queues a range. A backfill range is header-only and waits behind live ones (CONCEPT.md D32). */
|
||||
async queue(group: Group, startId: bigint, endId: bigint, { backfill = false } = {}) {
|
||||
const now = new Date()
|
||||
const [row] = await db
|
||||
.table('fetch_ranges')
|
||||
.insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), backfill, created_at: now, updated_at: now })
|
||||
.returning(['id', 'requeues'])
|
||||
await this.addJob(group.name, Number(row.id), 0, startId, endId, backfill)
|
||||
}
|
||||
|
||||
private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint, backfill: boolean) {
|
||||
await QueueService.nntpFetchQueue.add(
|
||||
'fetch-headers',
|
||||
{ groupName, startId: startId.toString(), endId: endId.toString(), rangeId, ...(backfill ? { backfill: true } : {}) },
|
||||
// The job id changes per attempt: BullMQ ignores an add() for an id it still knows.
|
||||
{ jobId: this.jobId(rangeId, requeues), ...(backfill ? { priority: indexerConfig.backfill.priority } : {}) }
|
||||
)
|
||||
}
|
||||
|
||||
private jobId(rangeId: number, requeues: number) {
|
||||
return `range-${rangeId}-${requeues}`
|
||||
}
|
||||
|
||||
async markDone(rangeId: number) {
|
||||
await db.from('fetch_ranges').where('id', rangeId).update({ status: 'done', error: null, updated_at: new Date() })
|
||||
}
|
||||
|
||||
async markFailed(rangeId: number, error: string) {
|
||||
await db.from('fetch_ranges').where('id', rangeId).update({ status: 'failed', error, updated_at: new Date() })
|
||||
}
|
||||
|
||||
/** Re-queues failed ranges, and queued ranges whose job no longer exists. */
|
||||
async requeue(group: Group): Promise<number> {
|
||||
const now = Date.now()
|
||||
const candidates = await db
|
||||
.from('fetch_ranges')
|
||||
.where('group_id', group.id)
|
||||
.where('requeues', '<', MAX_REQUEUES)
|
||||
.where((q) =>
|
||||
q
|
||||
.where((f) => f.where('status', 'failed').where('updated_at', '<', new Date(now - REQUEUE_AFTER_MS)))
|
||||
.orWhere((l) => l.where('status', 'queued').where('updated_at', '<', new Date(now - LOST_AFTER_MS)))
|
||||
)
|
||||
.orderBy('start_id')
|
||||
|
||||
let requeued = 0
|
||||
for (const range of candidates) {
|
||||
if (range.status === 'queued') {
|
||||
const job = await QueueService.nntpFetchQueue.getJob(this.jobId(Number(range.id), range.requeues))
|
||||
const state = job ? await job.getState() : 'missing'
|
||||
if (state !== 'missing' && state !== 'failed' && state !== 'unknown') continue
|
||||
}
|
||||
await db
|
||||
.from('fetch_ranges')
|
||||
.where('id', range.id)
|
||||
.update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() })
|
||||
await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id), Boolean(range.backfill))
|
||||
requeued++
|
||||
logger.warn(
|
||||
`Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}`
|
||||
)
|
||||
}
|
||||
return requeued
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves `groups.last_indexed_id` up to just below the first live range
|
||||
* that is not done yet (backfill ranges lie below it and don't count), and
|
||||
* forgets done ranges. Returns the new watermark.
|
||||
*/
|
||||
async advanceWatermark(group: Group): Promise<bigint> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce(
|
||||
(SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done' AND NOT backfill),
|
||||
g.last_queued_id))
|
||||
WHERE g.id = ? RETURNING last_indexed_id`,
|
||||
[group.id]
|
||||
)
|
||||
await db.from('fetch_ranges').where('group_id', group.id).where('status', 'done').delete()
|
||||
return BigInt(rows[0].last_indexed_id)
|
||||
}
|
||||
|
||||
/** Ranges that failed for good (for monitoring). */
|
||||
async abandoned(group: Group) {
|
||||
return db
|
||||
.from('fetch_ranges')
|
||||
.where('group_id', group.id)
|
||||
.where('status', 'failed')
|
||||
.where('requeues', '>=', MAX_REQUEUES)
|
||||
.count('* as count')
|
||||
.first()
|
||||
}
|
||||
}
|
||||
|
||||
export default new FetchRangeService()
|
||||
@@ -0,0 +1,60 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** files ids per slice: a few seconds of reads from the HDD. */
|
||||
const SLICE = 100_000
|
||||
|
||||
/**
|
||||
* The file_keys filter (LOG.md 2026-10-06 (7)): a 64-bit hash of every
|
||||
* file_key. The insert trigger adds new keys; `fill` copies the keys of
|
||||
* files from before, in id order and a slice at a time, and marks the
|
||||
* filter complete when it reaches the files that existed when it started
|
||||
* (everything after that has been added by the trigger, which was live
|
||||
* first). FileStore uses the filter only once it is complete.
|
||||
*/
|
||||
class FileKeys {
|
||||
private completeSince = 0
|
||||
|
||||
/** Whether every file's key is in file_keys; cached once true. */
|
||||
async complete(): Promise<boolean> {
|
||||
if (this.completeSince) return true
|
||||
const row = await db.from('file_keys_fill').where('id', 1).first()
|
||||
if (row?.complete) this.completeSince = Date.now()
|
||||
return Boolean(row?.complete)
|
||||
}
|
||||
|
||||
/** Forgets the cached state (tests). */
|
||||
forget() {
|
||||
this.completeSince = 0
|
||||
}
|
||||
|
||||
/** Copies keys until `budgetMs` has passed or the fill is done; returns the files read and whether it's complete. */
|
||||
async fill(budgetMs: number): Promise<{ read: number; complete: boolean; nextId: number; untilId: number }> {
|
||||
const started = Date.now()
|
||||
let state = await db.from('file_keys_fill').where('id', 1).first()
|
||||
if (!state) return { read: 0, complete: false, nextId: 0, untilId: 0 }
|
||||
if (state.until_id === null) {
|
||||
// Everything inserted from here on goes through the trigger (live since the migration).
|
||||
const [{ max }] = await db.from('files').max('id as max')
|
||||
await db.from('file_keys_fill').where('id', 1).update({ until_id: Number(max ?? 0), updated_at: new Date() })
|
||||
state = await db.from('file_keys_fill').where('id', 1).first()
|
||||
}
|
||||
let next = Number(state.next_id)
|
||||
const until = Number(state.until_id)
|
||||
let read = 0
|
||||
while (next <= until && Date.now() - started < budgetMs) {
|
||||
const to = Math.min(until, next + SLICE - 1)
|
||||
const result = await db.rawQuery(
|
||||
`INSERT INTO file_keys (key_hash)
|
||||
SELECT DISTINCT hashtextextended(file_key, 0) FROM files WHERE id BETWEEN ? AND ? AND file_key IS NOT NULL
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[next, to]
|
||||
)
|
||||
read += Number(result.rowCount ?? 0)
|
||||
next = to + 1
|
||||
await db.from('file_keys_fill').where('id', 1).update({ next_id: next, complete: next > until, updated_at: new Date() })
|
||||
}
|
||||
return { read, complete: next > until, nextId: next, untilId: until }
|
||||
}
|
||||
}
|
||||
|
||||
export default new FileKeys()
|
||||
@@ -0,0 +1,50 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/** Ids per partition (as the migration that partitioned files: …150). */
|
||||
export const FILE_PARTITION_SIZE = 4_000_000
|
||||
/**
|
||||
* Ids kept covered beyond the newest file: four partitions (16M ids, about a day of
|
||||
* seeding), so a few sweeps that can't create one are harmless.
|
||||
*/
|
||||
const AHEAD = 4 * FILE_PARTITION_SIZE
|
||||
|
||||
/**
|
||||
* `files` is partitioned by id (CONCEPT.md D39). An insert with an id no
|
||||
* partition covers fails, so gc:scheduler calls ensure() every sweep to keep
|
||||
* four partitions ready ahead of the newest id. Each new partition gets its
|
||||
* own unique index on file_key (FileStore.insert checks the older ones),
|
||||
* in the parent's tablespace (the HDD on homelab).
|
||||
*/
|
||||
class FilePartitions {
|
||||
async ensure(): Promise<string[]> {
|
||||
const created: string[] = []
|
||||
while (true) {
|
||||
// The sequence, not max(id): cheaper, and it counts ids handed out but not committed yet.
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT (SELECT last_value FROM files_id_seq) AS max_id,
|
||||
-- ''{0,1}, not ''?: knex reads ? as a binding.
|
||||
max((regexp_match(pg_get_expr(c.relpartbound, c.oid), 'TO \\(''{0,1}(\\d+)''{0,1}\\)'))[1]::bigint) AS upper
|
||||
FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid
|
||||
WHERE i.inhparent = 'files'::regclass`
|
||||
)
|
||||
const maxId = Number(rows[0].max_id)
|
||||
const upper = Number(rows[0].upper)
|
||||
if (upper - maxId >= AHEAD) return created
|
||||
const name = `files_p${upper}`
|
||||
// CREATE … PARTITION OF locks files and its partitions one by one, which deadlocked with
|
||||
// file inserts (11 failed sweeps on 2026-10-07). Taking the parent's lock first, in one
|
||||
// statement, can only wait; after 10 s it gives up until the next sweep.
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`SET LOCAL lock_timeout = '10s'`)
|
||||
await trx.rawQuery(`LOCK TABLE files IN ACCESS EXCLUSIVE MODE`)
|
||||
await trx.rawQuery(`CREATE TABLE ${name} PARTITION OF files FOR VALUES FROM (${upper}) TO (${upper + FILE_PARTITION_SIZE})`)
|
||||
await trx.rawQuery(`ALTER TABLE ${name} ADD CONSTRAINT ${name}_file_key_unique UNIQUE (file_key)`)
|
||||
})
|
||||
logger.info(`files: partition ${name} created (ids ${upper}–${upper + FILE_PARTITION_SIZE - 1}).`)
|
||||
created.push(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new FilePartitions()
|
||||
@@ -0,0 +1,138 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import FileKeys from '#services/FileKeys'
|
||||
import type { QueryClientContract } from '@adonisjs/lucid/types/database'
|
||||
import { packSegments, unpackSegments, type Segment } from '#lib/nzb/segments'
|
||||
|
||||
/**
|
||||
* The only code that knows how `files` keeps its segment lists (CONCEPT.md
|
||||
* D31): compressed in `files.segments`, with the first segment's message-id
|
||||
* in the indexed `first_message_id` for lookups. Read them with
|
||||
* `segmentsOf(file)`; write new files with `FileStore.insert`.
|
||||
*/
|
||||
|
||||
/** A new `files` row, with its segments as a list. */
|
||||
export type NewFile = Record<string, unknown> & { segments: Segment[] }
|
||||
|
||||
const unpacked = new WeakMap<object, Segment[]>()
|
||||
|
||||
/** The segments of a `files` row (a row from `db.from('files')`), in order. */
|
||||
export function segmentsOf(file: { segments?: Buffer | Segment[] | null }): Segment[] {
|
||||
const { segments } = file
|
||||
if (!segments) return []
|
||||
if (Array.isArray(segments)) return segments
|
||||
let list = unpacked.get(file)
|
||||
if (!list) {
|
||||
list = unpackSegments(segments)
|
||||
unpacked.set(file, list)
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
/** Stored form of a segment list, for the File model's column. */
|
||||
export function storedSegments(segments: Segment[]) {
|
||||
return packSegments(segments)
|
||||
}
|
||||
|
||||
export function loadedSegments(stored: Buffer | null): Segment[] {
|
||||
return stored ? unpackSegments(stored) : []
|
||||
}
|
||||
|
||||
/** What inspection learns about a file, kept beside `files` (LOG.md 2026-10-08 (14)). */
|
||||
export interface FileDetails {
|
||||
content_type?: string | null
|
||||
archive?: unknown
|
||||
}
|
||||
|
||||
const DETAIL_COLUMNS = ['content_type', 'archive'] as const
|
||||
|
||||
/** Rows per INSERT, well below Postgres' 65,535 bind parameters (tests lower it). */
|
||||
export const insertLimits = { rows: 2000 }
|
||||
|
||||
class FileStore {
|
||||
/** Sets some of a file's details, leaving the others as they are. */
|
||||
async setDetails(fileId: number, details: FileDetails, client: Pick<QueryClientContract, 'rawQuery'> = db) {
|
||||
const columns = DETAIL_COLUMNS.filter((c) => details[c] !== undefined)
|
||||
if (columns.length === 0) return
|
||||
const values = columns.map((c) => (c === 'archive' && details[c] !== null ? JSON.stringify(details[c]) : details[c]))
|
||||
await client.rawQuery(
|
||||
`INSERT INTO file_details (file_id, ${columns.join(', ')}) VALUES (?, ${columns.map(() => '?').join(', ')})
|
||||
ON CONFLICT (file_id) DO UPDATE SET ${columns.map((c) => `${c} = excluded.${c}`).join(', ')}`,
|
||||
[fileId, ...(values as any[])]
|
||||
)
|
||||
}
|
||||
|
||||
/** A file's details (empty when it has none). */
|
||||
async details(fileId: number): Promise<FileDetails> {
|
||||
return (await db.from('file_details').where('file_id', fileId).select('content_type', 'archive').first()) ?? {}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inserts files, skipping those whose `file_key` exists. Returns the rows
|
||||
* that were inserted (id, filename, file_key). `files` is partitioned by id
|
||||
* (CONCEPT.md D39) and each partition has its own unique file_key index:
|
||||
* once file_keys (a 64-bit hash of every key) is complete, it says which keys
|
||||
* may exist, and only those are looked up in the partitions; two
|
||||
* writers adding the same new key at once meet in the newest partition's index.
|
||||
*/
|
||||
async insert(
|
||||
files: NewFile[],
|
||||
client: Pick<QueryClientContract, 'knexQuery' | 'rawQuery'> = db
|
||||
): Promise<{ id: number; filename: string; file_key: string | null }[]> {
|
||||
if (files.length === 0) return []
|
||||
const keys = [...new Set(files.map((file) => file.file_key).filter((key): key is string => typeof key === 'string'))]
|
||||
const known = new Set<string>()
|
||||
const filtered = await FileKeys.complete()
|
||||
for (let i = 0; i < keys.length; i += 5000) {
|
||||
const chunk = keys.slice(i, i + 5000)
|
||||
if (!filtered) {
|
||||
const found = await client.knexQuery().from('files').whereIn('file_key', chunk).select('file_key')
|
||||
for (const row of found) known.add(row.file_key)
|
||||
continue
|
||||
}
|
||||
const { rows } = await client.rawQuery(
|
||||
`SELECT k.key FROM unnest(?::text[]) AS k(key)
|
||||
WHERE EXISTS (SELECT 1 FROM file_keys f WHERE f.key_hash = hashtextextended(k.key, 0))`,
|
||||
[chunk]
|
||||
)
|
||||
const maybe = rows.map((row: { key: string }) => row.key)
|
||||
if (maybe.length === 0) continue
|
||||
const found = await client.knexQuery().from('files').whereIn('file_key', maybe).select('file_key')
|
||||
for (const row of found) known.add(row.file_key)
|
||||
}
|
||||
const seen = new Set<string>()
|
||||
const rows = files
|
||||
.filter((file) => {
|
||||
const key = file.file_key as string | null | undefined
|
||||
if (!key) return true
|
||||
if (known.has(key) || seen.has(key)) return false
|
||||
seen.add(key)
|
||||
return true
|
||||
})
|
||||
.map(({ segments, ...file }) => {
|
||||
const sorted = [...segments].sort((a, b) => a.number - b.number)
|
||||
return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
|
||||
})
|
||||
if (rows.length === 0) return []
|
||||
// Details given with a new file (e.g. a known content type) go to file_details.
|
||||
const details = rows.map((row: Record<string, unknown>) => {
|
||||
const d: FileDetails = {}
|
||||
for (const c of DETAIL_COLUMNS) if (c in row) (d as any)[c] = row[c]
|
||||
for (const c of DETAIL_COLUMNS) delete row[c]
|
||||
return d
|
||||
})
|
||||
// A statement takes at most 65,535 parameters: ~5,400 rows of 12 columns.
|
||||
const inserted: { id: number; filename: string; file_key: string | null }[] = []
|
||||
for (let i = 0; i < rows.length; i += insertLimits.rows) {
|
||||
const chunk = await client.knexQuery().table('files').insert(rows.slice(i, i + insertLimits.rows)).onConflict().ignore().returning(['id', 'filename', 'file_key'])
|
||||
for (const row of chunk) inserted.push({ id: Number(row.id), filename: row.filename, file_key: row.file_key ?? null })
|
||||
}
|
||||
const detailsByKey = new Map(rows.map((row: Record<string, unknown>, i: number) => [row.file_key ?? row.filename, details[i]]))
|
||||
for (const row of inserted) {
|
||||
const d = detailsByKey.get(row.file_key ?? row.filename)
|
||||
if (d && Object.keys(d).length > 0) await this.setDetails(row.id, d, client)
|
||||
}
|
||||
return inserted
|
||||
}
|
||||
}
|
||||
|
||||
export default new FileStore()
|
||||
@@ -0,0 +1,217 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
import { explored, signatureGroup, signatureKey, uploadTool } from '#lib/garbage/index'
|
||||
|
||||
/** What the filter needs to know about an upload before it is stored. */
|
||||
export interface Upload {
|
||||
/** The posted file name. */
|
||||
name: string
|
||||
/** Any of its articles' Message-IDs (one tool posts all of a file). */
|
||||
messageId: string | null | undefined
|
||||
groups: unknown
|
||||
/** Its release's key (the file key when it has none): kept or dropped as a whole. */
|
||||
releaseKey: string
|
||||
}
|
||||
|
||||
/** Contents that can't name a random-named release without a key (LOG.md 2026-10-08 (5)). */
|
||||
const CONTAINERS = ['rclone', 'nzbe', 'yencbin']
|
||||
|
||||
/** How long the dropped signatures are cached per process. */
|
||||
const CACHE_MS = 5 * 60 * 1000
|
||||
|
||||
export function isRandomName(name: string) {
|
||||
return looksRandom(fileStem(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* Garbage: random-named uploads that never get a real name and have no
|
||||
* known password (CONCEPT.md D42). Two filters:
|
||||
* - before storing (seeding, ingest): random-named uploads whose signature
|
||||
* (posting tool, first group) is dropped, or whose release was dropped
|
||||
* after its probe, aren't stored;
|
||||
* - after a probe: a random-named release whose probes are all done, that
|
||||
* is still without a real name, password, names from its files or spot,
|
||||
* and is encrypted, a container or missing its first article, is deleted.
|
||||
* The dropped signatures are learned from a sample of probed releases
|
||||
* (learn(), gc:scheduler).
|
||||
*/
|
||||
class GarbageFilter {
|
||||
private dropped: Set<string> | null = null
|
||||
private loadedAt = 0
|
||||
|
||||
/** The dropped signatures, cached for CACHE_MS. */
|
||||
async droppedSignatures(): Promise<Set<string>> {
|
||||
if (this.dropped && Date.now() - this.loadedAt < CACHE_MS) return this.dropped
|
||||
const rows = await db.from('upload_signatures').where('dropped', true).select('tool', 'group_name')
|
||||
this.dropped = new Set(rows.map((row) => signatureKey(row.tool, row.group_name)))
|
||||
this.loadedAt = Date.now()
|
||||
return this.dropped
|
||||
}
|
||||
|
||||
/** Forgets the cached signatures (after learn(), and in tests). */
|
||||
reset() {
|
||||
this.dropped = null
|
||||
}
|
||||
|
||||
/** Whether a random-named upload's signature is dropped and it isn't in the explored share. */
|
||||
private droppedBySignature(upload: Upload, signatures: Set<string>) {
|
||||
if (signatures.size === 0) return false
|
||||
if (!signatures.has(signatureKey(uploadTool(upload.messageId), signatureGroup(upload.groups)))) return false
|
||||
return !explored(upload.releaseKey, indexerConfig.garbage.explorePercent)
|
||||
}
|
||||
|
||||
/**
|
||||
* The items worth storing: drops random-named uploads of dropped signatures
|
||||
* (but explorePercent of their releases) and of dropped releases.
|
||||
*/
|
||||
async keep<T>(items: T[], upload: (item: T) => Upload): Promise<{ kept: T[]; dropped: number }> {
|
||||
if (!indexerConfig.garbage.enabled || items.length === 0) return { kept: items, dropped: 0 }
|
||||
const signatures = await this.droppedSignatures()
|
||||
// Per release, kept as a whole when any of its files has a readable name. A file is
|
||||
// looked at once: an ingest batch has many articles of each.
|
||||
const verdicts = new Map<string, 'keep' | 'drop' | 'check'>()
|
||||
const seen = new Set<string>()
|
||||
for (const item of items) {
|
||||
const u = upload(item)
|
||||
if (verdicts.get(u.releaseKey) === 'keep' || seen.has(`${u.releaseKey}\n${u.name}`)) continue
|
||||
seen.add(`${u.releaseKey}\n${u.name}`)
|
||||
if (!isRandomName(u.name)) verdicts.set(u.releaseKey, 'keep')
|
||||
else if (!verdicts.has(u.releaseKey)) verdicts.set(u.releaseKey, this.droppedBySignature(u, signatures) ? 'drop' : 'check')
|
||||
}
|
||||
const check = [...verdicts].filter(([, v]) => v === 'check').map(([key]) => key)
|
||||
for (let i = 0; i < check.length; i += 5000) {
|
||||
const rows = await db.from('dropped_releases').whereIn('release_key', check.slice(i, i + 5000)).select('release_key')
|
||||
for (const row of rows) verdicts.set(row.release_key, 'drop')
|
||||
}
|
||||
const kept = items.filter((item) => verdicts.get(upload(item).releaseKey) !== 'drop')
|
||||
return { kept, dropped: items.length - kept.length }
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether a stored file's probe can be skipped: a random-named file of a
|
||||
* dropped signature, outside the explored share (files stored before its
|
||||
* signature was dropped). Its binary stops waiting for inspection.
|
||||
*/
|
||||
async skipProbe(file: { id: number | string; filename: string; real_name?: string | null; first_message_id?: string | null; groups: unknown; release_id?: number | string | null }) {
|
||||
if (!indexerConfig.garbage.enabled || file.real_name || !isRandomName(file.filename)) return false
|
||||
const signatures = await this.droppedSignatures()
|
||||
if (!signatures.has(signatureKey(uploadTool(file.first_message_id), signatureGroup(file.groups)))) return false
|
||||
const release = file.release_id ? await db.from('releases').where('id', file.release_id).select('release_key').first() : null
|
||||
if (explored(release?.release_key ?? String(file.id), indexerConfig.garbage.explorePercent)) return false
|
||||
await db.from('binaries').where('file_id', file.id).where('awaiting_inspection', true).update({ awaiting_inspection: null })
|
||||
return true
|
||||
}
|
||||
|
||||
/**
|
||||
* After a file's probe has been applied: notes its release for learning
|
||||
* (a sample) and deletes the release when it is garbage. Never throws:
|
||||
* the probe's result is stored already. Returns why the release was
|
||||
* deleted, or null.
|
||||
*/
|
||||
async afterProbe(fileId: number, contentType: string | null): Promise<string | null> {
|
||||
try {
|
||||
const file = await db.from('files').where('id', fileId).select('id', 'filename', 'real_name', 'first_message_id', 'groups', 'release_id').first()
|
||||
if (!file?.release_id || !isRandomName(String(file.filename))) return null
|
||||
const releaseId = Number(file.release_id)
|
||||
if (releaseId % indexerConfig.garbage.probeSample === 0) {
|
||||
await db.rawQuery(`INSERT INTO signature_probes (release_id, tool, group_name) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [
|
||||
releaseId,
|
||||
uploadTool(file.first_message_id),
|
||||
signatureGroup(file.groups),
|
||||
])
|
||||
}
|
||||
if (!indexerConfig.garbage.enabled) return null
|
||||
return await this.dropIfGarbage(releaseId, contentType)
|
||||
} catch (error: any) {
|
||||
logger.warn(`Garbage check after probing file ${fileId}: ${error.message}`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
private async dropIfGarbage(releaseId: number, contentType: string | null): Promise<string | null> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT r.release_key, r.container, r.encrypted,
|
||||
is_real_release_name(r.name) OR r.password IS NOT NULL OR coalesce(r.search_files, '') <> ''
|
||||
OR EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id) AS named,
|
||||
EXISTS (SELECT 1 FROM binaries b WHERE b.release_id = r.id AND b.awaiting_inspection) AS waiting
|
||||
FROM releases r WHERE r.id = ?`,
|
||||
[releaseId]
|
||||
)
|
||||
const release = rows[0]
|
||||
if (!release || release.named || release.waiting) return null
|
||||
const reason = CONTAINERS.includes(release.container) ? release.container : release.encrypted === true ? 'encrypted' : contentType === 'missing' ? 'missing' : null
|
||||
if (!reason) return null
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`INSERT INTO dropped_releases (release_key, reason) VALUES (?, ?) ON CONFLICT DO NOTHING`, [release.release_key, reason])
|
||||
await trx.rawQuery(`DELETE FROM binaries WHERE release_id = ?`, [releaseId])
|
||||
await trx.rawQuery(`DELETE FROM files WHERE release_id = ?`, [releaseId])
|
||||
await trx.rawQuery(`DELETE FROM releases WHERE id = ?`, [releaseId])
|
||||
})
|
||||
return reason
|
||||
}
|
||||
|
||||
/**
|
||||
* Learns the signatures from the probed releases of the last windowDays:
|
||||
* dropped when at least minProbes are known and at most maxNamedRate got a
|
||||
* real name, password, names from their files or a spot. A signature with
|
||||
* fewer probes keeps its verdict (a dropped one only gets explored probes).
|
||||
* Releases deleted as garbage count as unnamed. Returns the signatures
|
||||
* whose verdict changed.
|
||||
*/
|
||||
async learn(): Promise<{ signatures: number; dropped: number; changed: string[] }> {
|
||||
const { minProbes, maxNamedRate, windowDays } = indexerConfig.garbage
|
||||
await db.rawQuery(`DELETE FROM signature_probes WHERE probed_at < now() - make_interval(days => ?)`, [windowDays])
|
||||
const before = new Map((await db.from('upload_signatures').select('tool', 'group_name', 'dropped')).map((row) => [signatureKey(row.tool, row.group_name), row.dropped]))
|
||||
await db.rawQuery(
|
||||
`INSERT INTO upload_signatures (tool, group_name, probed, named, dropped, updated_at)
|
||||
SELECT tool, group_name, probed, named, probed >= ?::int AND named <= probed * ?::float8, now()
|
||||
FROM (
|
||||
SELECT p.tool, p.group_name, count(*)::int AS probed,
|
||||
(count(*) FILTER (WHERE is_real_release_name(r.name) OR r.password IS NOT NULL OR coalesce(r.search_files, '') <> ''
|
||||
OR EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id)))::int AS named
|
||||
FROM signature_probes p LEFT JOIN releases r ON r.id = p.release_id
|
||||
GROUP BY p.tool, p.group_name
|
||||
) s
|
||||
ON CONFLICT (tool, group_name) DO UPDATE SET
|
||||
probed = excluded.probed, named = excluded.named, updated_at = now(),
|
||||
dropped = CASE WHEN excluded.probed >= ?::int THEN excluded.dropped ELSE upload_signatures.dropped END`,
|
||||
[minProbes, maxNamedRate, minProbes]
|
||||
)
|
||||
const after = await db.from('upload_signatures').select('tool', 'group_name', 'dropped', 'probed', 'named')
|
||||
const changed = after
|
||||
.filter((row) => (before.get(signatureKey(row.tool, row.group_name)) ?? false) !== row.dropped)
|
||||
.map((row) => `${row.tool} ${row.group_name || '(no group)'}: ${row.dropped ? 'dropped' : 'kept'} (${row.named}/${row.probed} named)`)
|
||||
this.reset()
|
||||
return { signatures: after.length, dropped: after.filter((row) => row.dropped).length, changed }
|
||||
}
|
||||
|
||||
/**
|
||||
* Fills signature_probes from a sample of files probed before it existed
|
||||
* (garbage:learn --seed): random-named files with a content type, from
|
||||
* TABLESAMPLE SYSTEM (percent) of files. Returns the releases added.
|
||||
*/
|
||||
async seed(percent: number): Promise<number> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT ON (f.release_id) f.release_id, f.first_message_id, f.groups
|
||||
FROM files f TABLESAMPLE SYSTEM (?) JOIN file_details d ON d.file_id = f.id
|
||||
WHERE d.content_type IS NOT NULL AND f.release_id IS NOT NULL AND NOT is_readable_name(f.filename)`,
|
||||
[percent]
|
||||
)
|
||||
let added = 0
|
||||
for (let i = 0; i < rows.length; i += 2000) {
|
||||
const chunk = rows.slice(i, i + 2000)
|
||||
const result = await db.rawQuery(
|
||||
`INSERT INTO signature_probes (release_id, tool, group_name)
|
||||
SELECT * FROM unnest(?::bigint[], ?::text[], ?::text[]) ON CONFLICT DO NOTHING`,
|
||||
[chunk.map((row: any) => row.release_id), chunk.map((row: any) => uploadTool(row.first_message_id)), chunk.map((row: any) => signatureGroup(row.groups))]
|
||||
)
|
||||
added += result.rowCount ?? 0
|
||||
}
|
||||
return added
|
||||
}
|
||||
}
|
||||
|
||||
export default new GarbageFilter()
|
||||
@@ -0,0 +1,85 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/** Only indexes at least this big are worth checking. */
|
||||
const MIN_BYTES = 50 * 1024 * 1024
|
||||
/** Leaf pages less full than this (percent) mean the index is mostly empty space. */
|
||||
const MAX_LEAF_DENSITY = 50
|
||||
|
||||
export interface Rebuilt {
|
||||
index: string
|
||||
density: number
|
||||
before: number
|
||||
after: number
|
||||
}
|
||||
|
||||
/**
|
||||
* GC deletes millions of rows (single-article binaries, copied parts, empty
|
||||
* releases). Vacuum makes their index entries reusable but doesn't compact
|
||||
* the index, so B-trees on the churning tables end up 10–20% full (LOG.md
|
||||
* 2026-10-01). Rebuilding them online brings them back to size (CONCEPT.md
|
||||
* D33). Needs the pgstattuple extension.
|
||||
*/
|
||||
class IndexMaintenance {
|
||||
async rebuildBloated({ dryRun = false } = {}): Promise<Rebuilt[]> {
|
||||
const { rows: extension } = await db.rawQuery(`SELECT 1 FROM pg_extension WHERE extname = 'pgstattuple'`)
|
||||
if (extension.length === 0) {
|
||||
logger.warn('Index maintenance skipped: the pgstattuple extension is not installed.')
|
||||
return []
|
||||
}
|
||||
if (!dryRun) await this.dropLeftovers()
|
||||
|
||||
const { rows: candidates } = await db.rawQuery(
|
||||
`SELECT c.relname AS index, pg_relation_size(c.oid) AS bytes
|
||||
FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_am am ON am.oid = c.relam
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE am.amname = 'btree' AND n.nspname = 'public' AND i.indisvalid AND pg_relation_size(c.oid) >= ?
|
||||
ORDER BY 2 DESC`,
|
||||
[MIN_BYTES]
|
||||
)
|
||||
const rebuilt: Rebuilt[] = []
|
||||
for (const { index, bytes } of candidates) {
|
||||
const { rows } = await db.rawQuery(`SELECT avg_leaf_density FROM pgstatindex(?)`, [index])
|
||||
const density = Number(rows[0].avg_leaf_density)
|
||||
if (density >= MAX_LEAF_DENSITY) continue
|
||||
if (!dryRun && !(await this.reindex(index))) continue
|
||||
const { rows: after } = await db.rawQuery(`SELECT pg_relation_size(?::regclass) AS bytes`, [index])
|
||||
const result = { index, density, before: Number(bytes), after: Number(after[0].bytes) }
|
||||
rebuilt.push(result)
|
||||
logger.info(`Index ${index}: ${density.toFixed(0)}% full, ${mb(result.before)} → ${mb(result.after)}${dryRun ? ' (dry run)' : ''}.`)
|
||||
}
|
||||
return rebuilt
|
||||
}
|
||||
|
||||
/**
|
||||
* REINDEX CONCURRENTLY can lose a deadlock against busy writers (seen on
|
||||
* binaries during a backfill). Its half-built copy is dropped and it is
|
||||
* tried once more; a second failure skips the index until the next run.
|
||||
*/
|
||||
/** Invalid copies an interrupted REINDEX CONCURRENTLY left behind (`<index>_ccnew`, `_ccnew1`, …). */
|
||||
private async dropLeftovers(index = '') {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE NOT i.indisvalid AND c.relname LIKE ? || '%\\_ccnew%'`,
|
||||
[index]
|
||||
)
|
||||
for (const { relname } of rows) await db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS "${relname}"`)
|
||||
}
|
||||
|
||||
private async reindex(index: string): Promise<boolean> {
|
||||
for (let attempt = 1; attempt <= 2; attempt++) {
|
||||
try {
|
||||
await db.rawQuery(`REINDEX INDEX CONCURRENTLY "${index}"`)
|
||||
return true
|
||||
} catch (error: any) {
|
||||
logger.warn(`Rebuilding index ${index} failed (attempt ${attempt}): ${error.message}`)
|
||||
await this.dropLeftovers(index)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
const mb = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`
|
||||
|
||||
export default new IndexMaintenance()
|
||||
@@ -0,0 +1,424 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import type { QueryClientContract } from '@adonisjs/lucid/types/database'
|
||||
import type { Knex } from 'knex'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NameService from '#services/NameService'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
import type { IngestSegment } from '#lib/ingest/segment'
|
||||
|
||||
// Header parsing is pure and lives in lib, so seeding nodes need no database (CONCEPT.md D37).
|
||||
export { segmentFromHeader, segmentFromYenc, type IngestSegment, type OverviewHeader } from '#lib/ingest/segment'
|
||||
|
||||
export interface CompletedBinary {
|
||||
id: number
|
||||
fileKey: string
|
||||
}
|
||||
|
||||
const BINARY_CHUNK = 1000
|
||||
/** last_seen_at is only rewritten when it is older than this (it drives GC at 48 h). */
|
||||
const TOUCH_MS = 10 * 60 * 1000
|
||||
|
||||
type Merge = 'least' | 'coalesce' | 'touch'
|
||||
|
||||
/**
|
||||
* Inserts rows keyed by `key` and returns their ids. Existing rows are not
|
||||
* locked unless something in them changes (`least`: an earlier value,
|
||||
* `coalesce`: a value where there was none, `touch`: a timestamp more than
|
||||
* TOUCH_MS newer). Parallel fetch jobs see mostly the same rows, and
|
||||
* `INSERT … ON CONFLICT DO UPDATE` made them wait on each other for every one.
|
||||
*/
|
||||
async function upsertKeyed(
|
||||
knex: Knex,
|
||||
table: 'binaries' | 'releases',
|
||||
key: string,
|
||||
rows: Record<string, unknown>[],
|
||||
merge: Record<string, [Merge, string]>
|
||||
): Promise<Map<string, number>> {
|
||||
const ids = new Map<string, number>()
|
||||
if (rows.length === 0) return ids
|
||||
const inserted: Record<string, any>[] = await knex(table).insert(rows).onConflict(key).ignore().returning(['id', key])
|
||||
for (const row of inserted) ids.set(row[key], Number(row.id))
|
||||
|
||||
const incoming = new Map(rows.filter((r) => !ids.has(r[key] as string)).map((r) => [r[key] as string, r]))
|
||||
if (incoming.size === 0) return ids
|
||||
const columns = Object.keys(merge)
|
||||
const existing: Record<string, any>[] = await knex(table).whereIn(key, [...incoming.keys()]).select('id', key, ...columns)
|
||||
|
||||
const changes: Record<string, unknown>[] = []
|
||||
for (const row of existing) {
|
||||
ids.set(row[key], Number(row.id))
|
||||
const next = incoming.get(row[key])!
|
||||
const changed = columns.some((column) => {
|
||||
const [kind] = merge[column]
|
||||
const current = row[column]
|
||||
const value = next[column]
|
||||
if (value === null || value === undefined) return false
|
||||
if (kind === 'coalesce') return current === null
|
||||
if (kind === 'least') return current === null || value < current
|
||||
return current === null || +(value as Date) - +current > TOUCH_MS
|
||||
})
|
||||
if (changed) changes.push({ id: Number(row.id), ...Object.fromEntries(columns.map((c) => [c, next[c] ?? null])) })
|
||||
}
|
||||
if (changes.length === 0) return ids
|
||||
|
||||
const set = columns
|
||||
.map((c) => {
|
||||
const [kind] = merge[c]
|
||||
const fn = kind === 'least' ? 'least' : kind === 'touch' ? 'greatest' : 'coalesce'
|
||||
return `${c} = ${fn}(t.${c}, v.${c})`
|
||||
})
|
||||
.join(', ')
|
||||
const types = columns.map((c) => `${c} ${merge[c][1]}`).join(', ')
|
||||
changes.sort((a, b) => (a.id as number) - (b.id as number))
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
await knex.raw(
|
||||
`UPDATE ${table} t SET ${set}
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, ${types})
|
||||
WHERE t.id = v.id`,
|
||||
[JSON.stringify(changes)]
|
||||
)
|
||||
return ids
|
||||
} catch (error: any) {
|
||||
// Rare now, but two jobs can still update overlapping rows in a different order.
|
||||
if (error.code !== '40P01' || attempt >= 3) throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
/** Completed binaries per worker:file job. */
|
||||
const FILE_BATCH = 100
|
||||
|
||||
/** Size-only matching below this is too ambiguous (nfo, sfv, small volumes). */
|
||||
export const PAR2_SIZE_ONLY_MIN = 1024 * 1024
|
||||
/** A par2 set and its files are posted together; don't match across days. */
|
||||
export const PAR2_MATCH_WINDOW_SECONDS = 12 * 60 * 60
|
||||
const PART_CHUNK = 2000
|
||||
|
||||
function chunks<T>(items: T[], size: number): T[][] {
|
||||
const result: T[][] = []
|
||||
for (let i = 0; i < items.length; i += size) result.push(items.slice(i, i + size))
|
||||
return result
|
||||
}
|
||||
|
||||
function validDate(date?: Date) {
|
||||
return date && !Number.isNaN(date.getTime()) ? date : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores segments in `binaries`/`parts` and queues every binary that became
|
||||
* complete exactly once. Idempotent: ingesting the same segments again is a
|
||||
* no-op. See CONCEPT.md (D12).
|
||||
*/
|
||||
class IngestService {
|
||||
async ingest(batch: IngestSegment[]): Promise<CompletedBinary[]> {
|
||||
// Garbage isn't stored at all: no release, binary or parts (CONCEPT.md D42).
|
||||
const { kept: segments } = await GarbageFilter.keep(batch, (s) => ({
|
||||
name: s.name,
|
||||
messageId: s.messageId,
|
||||
groups: s.groups,
|
||||
releaseKey: s.release?.key ?? s.fileKey,
|
||||
}))
|
||||
if (segments.length === 0) return []
|
||||
|
||||
// Short statements, no enclosing transaction: every step is idempotent, and
|
||||
// row locks held for a whole batch made parallel fetch jobs wait on each other.
|
||||
const binaryIds = await (async () => {
|
||||
const knex = db.connection().getWriteClient()
|
||||
const now = new Date()
|
||||
const releaseIds = await this.upsertReleases(knex, segments, now)
|
||||
|
||||
// One row per file_key (ON CONFLICT cannot touch a row twice per statement),
|
||||
// sorted so concurrent workers lock rows in the same order and cannot deadlock.
|
||||
const binaries = new Map<string, Record<string, unknown>>()
|
||||
for (const segment of segments) {
|
||||
const postedAt = validDate(segment.postedAt)
|
||||
const existing = binaries.get(segment.fileKey)
|
||||
if (existing) {
|
||||
if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
|
||||
existing.posted_at = postedAt
|
||||
}
|
||||
continue
|
||||
}
|
||||
binaries.set(segment.fileKey, {
|
||||
file_key: segment.fileKey,
|
||||
poster: segment.poster,
|
||||
subject: segment.subject,
|
||||
name: segment.name,
|
||||
total_parts: segment.totalParts,
|
||||
groups: JSON.stringify(segment.groups),
|
||||
posted_at: postedAt,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
release_id: segment.release ? releaseIds.get(segment.release.key) : null,
|
||||
size: segment.size ?? null,
|
||||
md5_16k: segment.md5_16k ?? null,
|
||||
})
|
||||
}
|
||||
const binaryRows = [...binaries.values()].sort((a, b) =>
|
||||
(a.file_key as string).localeCompare(b.file_key as string)
|
||||
)
|
||||
|
||||
const idsByKey = new Map<string, number>()
|
||||
for (const chunk of chunks(binaryRows, BINARY_CHUNK)) {
|
||||
const ids = await upsertKeyed(knex, 'binaries', 'file_key', chunk, {
|
||||
last_seen_at: ['touch', 'timestamptz'],
|
||||
posted_at: ['least', 'timestamptz'],
|
||||
release_id: ['coalesce', 'bigint'],
|
||||
md5_16k: ['coalesce', 'varchar'],
|
||||
})
|
||||
for (const [key, id] of ids) idsByKey.set(key, id)
|
||||
}
|
||||
|
||||
// A binary that is a file already has its segment list in `files`; its
|
||||
// header parts were dropped then. Crossposts in other groups, reposts
|
||||
// and backfill ranges keep bringing the same articles, and inserting
|
||||
// them again grew `parts` to 56M rows nobody reads (LOG.md 2026-10-04).
|
||||
const done = new Set<number>()
|
||||
const ids = [...idsByKey.values()]
|
||||
for (const chunk of chunks(ids, BINARY_CHUNK)) {
|
||||
const { rows } = await knex.raw('SELECT id FROM binaries WHERE id = ANY(?) AND file_id IS NOT NULL', [chunk])
|
||||
for (const row of rows) done.add(Number(row.id))
|
||||
}
|
||||
|
||||
const partRows = segments
|
||||
.map((segment) => ({
|
||||
binary_id: idsByKey.get(segment.fileKey)!,
|
||||
number: segment.number,
|
||||
message_id: segment.messageId,
|
||||
bytes: segment.bytes,
|
||||
article_number: segment.articleNumber?.toString() ?? null,
|
||||
pcrc32: segment.pcrc32 ?? null,
|
||||
part_size: segment.partSize ?? null,
|
||||
file_crc32: segment.fileCrc32 ?? null,
|
||||
md5_16k: segment.md5_16k ?? null,
|
||||
}))
|
||||
const weakRows = partRows.filter((row, i) => segments[i].weakIdentity && !(done.has(row.binary_id) && row.pcrc32 === null))
|
||||
// Body-derived parts (with a per-part CRC) are kept, as worker:file does.
|
||||
const newParts = partRows.filter((row) => !(done.has(row.binary_id) && row.pcrc32 === null))
|
||||
|
||||
newParts.sort((a, b) => a.binary_id - b.binary_id || a.number - b.number)
|
||||
// Parts and the binaries' stored-part counts in one statement, so the count can't miss a
|
||||
// part (LOG.md 2026-10-06 (7)); claimCompleted compares it instead of counting parts. NULL
|
||||
// (binaries from before part_count) is counted once: the statement's own rows aren't in
|
||||
// its snapshot, so they're added on top. Concurrent batches can lock binaries in a
|
||||
// different order: a deadlock rolls the whole statement back, and it is retried.
|
||||
for (const chunk of chunks(newParts, PART_CHUNK)) {
|
||||
const insert = knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore().returning('binary_id').toSQL()
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
await knex.raw(
|
||||
`WITH added AS (${insert.sql}),
|
||||
counted AS (SELECT binary_id, count(*)::int AS n FROM added GROUP BY binary_id)
|
||||
UPDATE binaries b SET part_count = CASE
|
||||
WHEN b.part_count IS NULL THEN (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) + c.n
|
||||
ELSE b.part_count + c.n END
|
||||
FROM counted c WHERE b.id = c.binary_id`,
|
||||
insert.bindings as any[]
|
||||
)
|
||||
break
|
||||
} catch (error: any) {
|
||||
if (error.code !== '40P01' || attempt >= 5) throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (weakRows.length > 0) await this.flagConflicts(db.connection(), weakRows)
|
||||
|
||||
return [...idsByKey.values()]
|
||||
})()
|
||||
|
||||
// par2 naming needs a size, which only body-derived segments have; for
|
||||
// header batches these queries were most of Postgres' work.
|
||||
const sized = [...new Set(segments.filter((s) => s.size != null).map((s) => s.fileKey))]
|
||||
if (sized.length > 0) {
|
||||
const sizedIds = (await db.from('binaries').whereIn('file_key', sized).select('id')).map((r) => Number(r.id))
|
||||
await this.matchPar2Releases(sizedIds)
|
||||
await NameService.resolve(sizedIds)
|
||||
}
|
||||
|
||||
// After commit, so the last writer of a binary always sees all of its parts.
|
||||
const completed = await this.claimCompleted(binaryIds)
|
||||
await this.enqueue(completed)
|
||||
return completed
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates or touches the releases referenced by `segments` and returns
|
||||
* their ids by release key. Also used by `release:rebuild`.
|
||||
*/
|
||||
async upsertReleases(
|
||||
knex: Knex,
|
||||
segments: Pick<IngestSegment, 'release' | 'poster' | 'groups' | 'postedAt' | 'weakIdentity'>[],
|
||||
now = new Date()
|
||||
): Promise<Map<string, number>> {
|
||||
const releases = new Map<string, Record<string, unknown>>()
|
||||
for (const segment of segments) {
|
||||
const release = segment.release
|
||||
if (!release) continue
|
||||
const postedAt = validDate(segment.postedAt)
|
||||
const existing = releases.get(release.key)
|
||||
if (existing) {
|
||||
if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
|
||||
existing.posted_at = postedAt
|
||||
}
|
||||
// Not every file carries the [i/M] counter (e.g. a par2 index): take it from any that does.
|
||||
if (existing.file_total === null && release.fileTotal) existing.file_total = release.fileTotal
|
||||
continue
|
||||
}
|
||||
releases.set(release.key, {
|
||||
release_key: release.key,
|
||||
name: release.name,
|
||||
// Obfuscating posters randomise the poster, so it says nothing about the release.
|
||||
poster: segment.weakIdentity ? null : segment.poster,
|
||||
file_total: release.fileTotal ?? null,
|
||||
source: release.source,
|
||||
groups: JSON.stringify(segment.groups),
|
||||
posted_at: postedAt,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
})
|
||||
}
|
||||
|
||||
const ids = new Map<string, number>()
|
||||
const rows = [...releases.values()].sort((a, b) =>
|
||||
(a.release_key as string).localeCompare(b.release_key as string)
|
||||
)
|
||||
for (const chunk of chunks(rows, BINARY_CHUNK)) {
|
||||
const result = await upsertKeyed(knex, 'releases', 'release_key', chunk, {
|
||||
last_seen_at: ['touch', 'timestamptz'],
|
||||
posted_at: ['least', 'timestamptz'],
|
||||
file_total: ['coalesce', 'integer'],
|
||||
source: ['coalesce', 'varchar'],
|
||||
})
|
||||
for (const [key, id] of result) ids.set(key, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
/**
|
||||
* Assigns body-derived binaries without a release and without a
|
||||
* fingerprint to a known par2 set by size: only when that size is listed
|
||||
* once in the set, the file is large, and exactly one set nearby lists it.
|
||||
* Binaries with a fingerprint are matched exactly by NameService instead.
|
||||
*/
|
||||
async matchPar2Releases(binaryIds: number[]) {
|
||||
if (binaryIds.length === 0) return
|
||||
// Listed-once first (index-only on release_files (release_id, size): most candidate
|
||||
// sizes are a set's equal-sized volumes), the release's date only for what's left;
|
||||
// OFFSET 0 keeps that order. Was: every size match's date and count, 18 s per batch.
|
||||
await db.rawQuery(
|
||||
`UPDATE binaries b SET release_id = m.release_id
|
||||
FROM (
|
||||
SELECT c.id, min(c.release_id) AS release_id
|
||||
FROM (
|
||||
SELECT b2.id, b2.posted_at, rf.release_id
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.size = b2.size
|
||||
WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM release_files x
|
||||
WHERE x.release_id = rf.release_id AND x.size = rf.size AND x.par2_file_id <> rf.par2_file_id)
|
||||
OFFSET 0
|
||||
) c
|
||||
JOIN releases r ON r.id = c.release_id
|
||||
WHERE abs(extract(epoch FROM (r.posted_at - c.posted_at))) < ?
|
||||
GROUP BY c.id
|
||||
HAVING count(DISTINCT c.release_id) = 1
|
||||
) m
|
||||
WHERE b.id = m.id`,
|
||||
[binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS]
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Flags binaries where a stored part has a different message-id and a
|
||||
* different part CRC than the incoming one: two files were merged under
|
||||
* one identity. Identical reposts (same CRC) are ignored.
|
||||
*/
|
||||
private async flagConflicts(
|
||||
trx: QueryClientContract,
|
||||
rows: {
|
||||
binary_id: number
|
||||
number: number
|
||||
message_id: string
|
||||
bytes: number
|
||||
article_number: string | null
|
||||
pcrc32: string | null
|
||||
part_size: number | null
|
||||
file_crc32: string | null
|
||||
md5_16k: string | null
|
||||
}[]
|
||||
) {
|
||||
const bindings = [
|
||||
rows.map((r) => r.binary_id),
|
||||
rows.map((r) => r.number),
|
||||
rows.map((r) => r.message_id),
|
||||
rows.map((r) => r.bytes),
|
||||
rows.map((r) => r.article_number),
|
||||
rows.map((r) => r.pcrc32),
|
||||
rows.map((r) => r.part_size),
|
||||
rows.map((r) => r.file_crc32),
|
||||
rows.map((r) => r.md5_16k),
|
||||
]
|
||||
// Keep the extra articles: SplitService separates the volumes from them.
|
||||
const conflicts = await trx.rawQuery(
|
||||
`INSERT INTO part_conflicts (binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
|
||||
SELECT v.binary_id, v.number, v.message_id, v.bytes, v.article_number, v.pcrc32, v.part_size, v.file_crc32, v.md5_16k
|
||||
FROM unnest(?::bigint[], ?::int[], ?::text[], ?::int[], ?::bigint[], ?::text[], ?::int[], ?::text[], ?::text[])
|
||||
AS v(binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
|
||||
JOIN parts p ON p.binary_id = v.binary_id AND p.number = v.number
|
||||
WHERE p.message_id <> v.message_id
|
||||
-- Same part CRC: the same data posted again (a repost), not another file.
|
||||
AND (p.pcrc32 IS NULL OR v.pcrc32 IS NULL OR p.pcrc32 <> v.pcrc32)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING binary_id`,
|
||||
bindings as any
|
||||
)
|
||||
const binaryIds: number[] = [
|
||||
...new Set<number>(conflicts.rows.map((r: { binary_id: string }) => Number(r.binary_id))),
|
||||
]
|
||||
if (binaryIds.length === 0) return
|
||||
const result = await trx.rawQuery(
|
||||
`UPDATE binaries SET conflicted_at = now()
|
||||
WHERE conflicted_at IS NULL AND id = ANY(?) RETURNING name`,
|
||||
[binaryIds]
|
||||
)
|
||||
for (const row of result.rows) {
|
||||
logger.warn(`Binary "${row.name}" has conflicting parts; it will be split into volumes.`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks complete binaries among `binaryIds` as claimed. The single UPDATE
|
||||
* locks each row, so concurrent callers never claim the same binary twice.
|
||||
*/
|
||||
async claimCompleted(binaryIds: number[]): Promise<CompletedBinary[]> {
|
||||
if (binaryIds.length === 0) return []
|
||||
const result = await db.rawQuery(
|
||||
`UPDATE binaries b SET completed_at = now()
|
||||
WHERE b.id = ANY(?) AND b.completed_at IS NULL AND b.conflicted_at IS NULL
|
||||
AND coalesce(b.part_count, (SELECT count(*) FROM parts p WHERE p.binary_id = b.id)) >= b.total_parts
|
||||
RETURNING b.id, b.file_key`,
|
||||
[binaryIds]
|
||||
)
|
||||
return result.rows.map((row: { id: string; file_key: string }) => ({
|
||||
id: Number(row.id),
|
||||
fileKey: row.file_key,
|
||||
}))
|
||||
}
|
||||
|
||||
async enqueue(completed: CompletedBinary[]) {
|
||||
if (completed.length === 0) return
|
||||
// A job per FILE_BATCH binaries: worker:file stores them with a few set-based queries (LOG.md 2026-10-06 (7)).
|
||||
// jobId from the first file_key: BullMQ ignores an add() for an id that already exists.
|
||||
const jobs = []
|
||||
for (let i = 0; i < completed.length; i += FILE_BATCH) {
|
||||
const batch = completed.slice(i, i + FILE_BATCH)
|
||||
jobs.push({ name: 'process-files', data: { binaryIds: batch.map((b) => b.id) }, opts: { jobId: `${batch[0].fileKey}-${batch.length}` } })
|
||||
}
|
||||
await QueueService.fileQueue.addBulk(jobs)
|
||||
}
|
||||
}
|
||||
|
||||
export default new IngestService()
|
||||
@@ -0,0 +1,66 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import env from '#start/env'
|
||||
import QueueService from '#services/QueueService'
|
||||
|
||||
/** Files per transaction when moving from the table to BullMQ. */
|
||||
const FEED_BATCH = 5000
|
||||
|
||||
/** Priorities: lower goes first. */
|
||||
export const INSPECTION_PRIORITY = { targeted: -1, normal: 0, low: 10 } as const
|
||||
|
||||
/**
|
||||
* The content inspection backlog (CONCEPT.md D21, D41; LOG.md 2026-10-07 (8)).
|
||||
* Producers add file ids to the inspection_queue table; feed() moves them
|
||||
* into BullMQ's collection-queue while fewer than INSPECTION_FEED_MAX jobs
|
||||
* wait there, where worker:collection and inspecting nodes take them as
|
||||
* before. Redis holds a working set instead of a backlog of days.
|
||||
*/
|
||||
class InspectionQueue {
|
||||
readonly feedMax = env.get('INSPECTION_FEED_MAX', 50_000)
|
||||
|
||||
/** Queues files for inspection; a file already queued keeps the higher (lower number) priority. */
|
||||
async add(fileIds: number[], priority: number = INSPECTION_PRIORITY.normal) {
|
||||
const ids = [...new Set(fileIds)]
|
||||
for (let i = 0; i < ids.length; i += 10_000) {
|
||||
await db.rawQuery(
|
||||
`INSERT INTO inspection_queue (file_id, priority)
|
||||
SELECT id, ?::smallint FROM unnest(?::bigint[]) AS id
|
||||
ON CONFLICT (file_id) DO UPDATE SET priority = least(inspection_queue.priority, excluded.priority)`,
|
||||
[priority, ids.slice(i, i + 10_000)]
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/** Tops BullMQ up to feedMax waiting jobs; returns how many were moved. */
|
||||
async feed(): Promise<number> {
|
||||
const queue = QueueService.collectionQueue
|
||||
const counts = await queue.getJobCounts('wait', 'prioritized')
|
||||
let room = this.feedMax - (counts.wait ?? 0) - (counts.prioritized ?? 0)
|
||||
let moved = 0
|
||||
while (room > 0) {
|
||||
const n = await db.transaction(async (trx) => {
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT file_id FROM inspection_queue ORDER BY priority, queued_at LIMIT ? FOR UPDATE SKIP LOCKED`,
|
||||
[Math.min(room, FEED_BATCH)]
|
||||
)
|
||||
if (rows.length === 0) return 0
|
||||
const ids = rows.map((row: { file_id: string }) => Number(row.file_id))
|
||||
// Added before the rows go: a failure leaves them queued (a job that went in anyway is deduplicated by its id).
|
||||
await queue.addBulk(ids.map((id: number) => ({ name: 'process-collection', data: { fileId: id }, opts: { jobId: `file-${id}` } })))
|
||||
await trx.rawQuery(`DELETE FROM inspection_queue WHERE file_id = ANY(?::bigint[])`, [ids])
|
||||
return ids.length
|
||||
})
|
||||
if (n === 0) break
|
||||
moved += n
|
||||
room -= n
|
||||
}
|
||||
return moved
|
||||
}
|
||||
|
||||
async size(): Promise<number> {
|
||||
const { rows } = await db.rawQuery(`SELECT count(*) AS n FROM inspection_queue`)
|
||||
return Number(rows[0].n)
|
||||
}
|
||||
}
|
||||
|
||||
export default new InspectionQueue()
|
||||
@@ -0,0 +1,128 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
|
||||
/**
|
||||
* A release that is nothing more than one randomly named file (its own
|
||||
* filename stem). Such a file moves into the par2 set that names it exactly;
|
||||
* real groupings (par2, RAR, NZB, spot, subject, post id) are left alone.
|
||||
*/
|
||||
export const WEAK_RELEASE_SQL = (column: string) =>
|
||||
`EXISTS (SELECT 1 FROM releases wr WHERE wr.id = ${column} AND wr.source IN ('stem', 'weak-stem')
|
||||
AND wr.file_count <= 1 AND (wr.name !~ '[._ -]' OR NOT is_readable_name(wr.name)))`
|
||||
|
||||
/**
|
||||
* Whether `candidate` (e.g. from a par2 listing) should replace `current`:
|
||||
* the current name is random, or a single token without separators
|
||||
* (mOzyyxCyYUj, F8lv3w5agMDI3; too short for looksRandom), while the
|
||||
* candidate reads like a release name.
|
||||
*/
|
||||
export function isBetterName(current: string, candidate: string): boolean {
|
||||
if (looksRandom(candidate) || !/[._ -]/.test(candidate)) return false
|
||||
return looksRandom(current) || !/[._ -]/.test(current)
|
||||
}
|
||||
|
||||
/** Smaller files (nfo, sfv) collide too easily on size alone. */
|
||||
const MIN_SIZE_ONLY = 64 * 1024
|
||||
|
||||
/**
|
||||
* Gives binaries (and their files) the real filename that a par2 set lists
|
||||
* for them. See CONCEPT.md (D21).
|
||||
* - Exact: size and MD5 of the first 16 KiB both match a par2 entry. This
|
||||
* also puts the binary in that par2 set's release.
|
||||
* - Size only, when the fingerprint is unknown: within the binary's own
|
||||
* release, only when exactly one listed file has that size.
|
||||
*/
|
||||
class NameService {
|
||||
async resolve(binaryIds: number[]): Promise<number> {
|
||||
if (binaryIds.length === 0) return 0
|
||||
const exact = await db.rawQuery(
|
||||
`UPDATE binaries b SET real_name = m.name,
|
||||
release_id = CASE WHEN b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')} THEN m.release_id ELSE b.release_id END
|
||||
FROM (
|
||||
SELECT DISTINCT ON (b2.id) b2.id, rf.name, rf.release_id
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.size = b2.size AND rf.md5_16k = b2.md5_16k
|
||||
WHERE b2.id = ANY(?)
|
||||
ORDER BY b2.id, rf.release_id DESC
|
||||
) m
|
||||
WHERE b.id = m.id AND (b.real_name IS DISTINCT FROM m.name OR b.release_id IS NULL
|
||||
OR (b.release_id <> m.release_id AND ${WEAK_RELEASE_SQL('b.release_id')}))`,
|
||||
[binaryIds]
|
||||
)
|
||||
const bySize = await db.rawQuery(
|
||||
`UPDATE binaries b SET real_name = m.name
|
||||
FROM (
|
||||
SELECT b2.id, min(rf.name) AS name
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.release_id = b2.release_id AND rf.size = b2.size
|
||||
WHERE b2.id = ANY(?) AND b2.real_name IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
|
||||
GROUP BY b2.id
|
||||
HAVING count(DISTINCT rf.name) = 1
|
||||
) m
|
||||
WHERE b.id = m.id`,
|
||||
[binaryIds, MIN_SIZE_ONLY]
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE files f SET real_name = b.real_name,
|
||||
release_id = CASE WHEN f.release_id IS NULL OR ${WEAK_RELEASE_SQL('f.release_id')} THEN coalesce(b.release_id, f.release_id) ELSE f.release_id END
|
||||
FROM binaries b
|
||||
WHERE b.id = ANY(?) AND b.file_id = f.id
|
||||
AND (f.real_name IS DISTINCT FROM b.real_name OR (f.release_id IS NULL AND b.release_id IS NOT NULL)
|
||||
OR (f.release_id <> b.release_id AND ${WEAK_RELEASE_SQL('f.release_id')}))`,
|
||||
[binaryIds]
|
||||
)
|
||||
return (exact.rowCount ?? 0) + (bySize.rowCount ?? 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* When a par2 set arrives: the binaries it can name, i.e. those matching
|
||||
* one of its entries exactly (size + first-16-KiB MD5) and those in its
|
||||
* release. Selecting every binary of the listed sizes instead picked up
|
||||
* hundreds of thousands of equal-sized volumes per set (LOG.md 2026-09-28 (18)).
|
||||
*/
|
||||
async resolveForRelease(releaseId: number): Promise<number> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT b.id FROM release_files rf
|
||||
JOIN binaries b ON b.md5_16k = rf.md5_16k AND b.size = rf.size
|
||||
WHERE rf.release_id = ?
|
||||
UNION
|
||||
SELECT id FROM binaries WHERE release_id = ? AND size IS NOT NULL`,
|
||||
[releaseId, releaseId]
|
||||
)
|
||||
return this.resolve(rows.map((r: { id: string }) => Number(r.id)))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Repair for data from before files moved into the par2 set that names them
|
||||
* (used by `release:rename`): regroups exact matches out of random-named
|
||||
* single-file releases, then renames par2 releases that still carry a
|
||||
* random name after their listing. No downloads.
|
||||
*/
|
||||
export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; renamed: number }> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT b.id FROM binaries b
|
||||
JOIN release_files rf ON rf.md5_16k = b.md5_16k AND rf.size = b.size
|
||||
WHERE b.release_id IS DISTINCT FROM rf.release_id
|
||||
AND (b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')})`
|
||||
)
|
||||
const ids = rows.map((r: { id: string }) => Number(r.id))
|
||||
for (let i = 0; i < ids.length; i += 1000) await nameService.resolve(ids.slice(i, i + 1000))
|
||||
|
||||
const { par2ReleaseName } = await import('#lib/release/index')
|
||||
const listings = await db.rawQuery(
|
||||
`SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r
|
||||
JOIN release_files rf ON rf.release_id = r.id WHERE r.source <> 'spot' GROUP BY r.id, r.name`
|
||||
)
|
||||
let renamed = 0
|
||||
for (const release of listings.rows) {
|
||||
const better = par2ReleaseName(release.names)
|
||||
if (!better || better === release.name || !isBetterName(release.name, better)) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
renamed++
|
||||
}
|
||||
return { regrouped: ids.length, renamed }
|
||||
}
|
||||
|
||||
const nameService = new NameService()
|
||||
export default nameService
|
||||
@@ -0,0 +1,141 @@
|
||||
import nntpConfig from '#config/nntp'
|
||||
import { NNTP } from "#lib/nntp/nntp_js";
|
||||
import { parseGroupResponse, type GroupInfo } from '#lib/nntp/group'
|
||||
|
||||
class NntpService {
|
||||
private readonly poolSize: number;
|
||||
private allConnections: Set<any>;
|
||||
private idleConnections: any[];
|
||||
private waiters: ((conn: any) => void)[];
|
||||
/** Background work (archive walks): served only when no one else waits. */
|
||||
private lowWaiters: ((conn: any) => void)[] = [];
|
||||
private createdCount: number;
|
||||
|
||||
constructor(poolSize = 10) {
|
||||
this.poolSize = poolSize;
|
||||
this.allConnections = new Set();
|
||||
this.idleConnections = [];
|
||||
this.waiters = [];
|
||||
this.createdCount = 0;
|
||||
}
|
||||
|
||||
private async _createConnection() {
|
||||
const conn = new NNTP(nntpConfig.host, nntpConfig.port);
|
||||
try {
|
||||
await conn.connect();
|
||||
if (nntpConfig.user) {
|
||||
await conn.login(nntpConfig.user, nntpConfig.password?.release());
|
||||
}
|
||||
} catch (error) {
|
||||
// Free the slot, otherwise a failed connect shrinks the pool for good.
|
||||
this.createdCount--;
|
||||
this._destroy(conn);
|
||||
throw error;
|
||||
}
|
||||
this.allConnections.add(conn);
|
||||
return conn;
|
||||
}
|
||||
|
||||
/** A connection from the pool; with `low`, after every waiting normal caller. */
|
||||
public async acquire(low = false) {
|
||||
if (this.idleConnections.length > 0) {
|
||||
return this.idleConnections.pop();
|
||||
}
|
||||
|
||||
if (this.createdCount < this.poolSize) {
|
||||
this.createdCount++;
|
||||
console.log(`Creating new connection (${this.createdCount}/${this.poolSize}).`);
|
||||
return this._createConnection();
|
||||
}
|
||||
|
||||
return new Promise(resolve => (low ? this.lowWaiters : this.waiters).push(resolve));
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns a connection to the pool. Pass `failed` when a command on it
|
||||
* threw: the connection may be dead (e.g. closed by the server while idle),
|
||||
* so it is discarded instead of being handed to the next job.
|
||||
*/
|
||||
public release(conn: any, failed = false) {
|
||||
if (failed) {
|
||||
this.discard(conn);
|
||||
return;
|
||||
}
|
||||
const resolve = this.nextWaiter();
|
||||
if (resolve) {
|
||||
resolve(conn);
|
||||
} else {
|
||||
this.idleConnections.push(conn);
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes a broken connection and frees its slot for a new one. */
|
||||
public discard(conn: any) {
|
||||
this.allConnections.delete(conn);
|
||||
this.createdCount--;
|
||||
this._destroy(conn);
|
||||
|
||||
// Someone is waiting for a slot: open a fresh connection for them.
|
||||
const resolve = this.nextWaiter();
|
||||
if (resolve) {
|
||||
this.createdCount++;
|
||||
this._createConnection().then(resolve, () => this.waiters.unshift(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
private nextWaiter() {
|
||||
return this.waiters.shift() ?? this.lowWaiters.shift();
|
||||
}
|
||||
|
||||
private _destroy(conn: any) {
|
||||
try {
|
||||
conn.sock?.destroy();
|
||||
} catch {
|
||||
// already closed
|
||||
}
|
||||
}
|
||||
|
||||
/** Selects a group and returns its correctly parsed article range. */
|
||||
public async selectGroup(conn: any, name: string): Promise<GroupInfo> {
|
||||
const { resp } = await conn.group(name)
|
||||
return parseGroupResponse(resp)
|
||||
}
|
||||
|
||||
/**
|
||||
* Rejects when `promise` hasn't settled within `ms`. nntp-js has no
|
||||
* timeouts: a connection the server stops answering (still ESTABLISHED,
|
||||
* nothing in flight) leaves its command waiting forever. The caller must
|
||||
* then release the connection as failed, so it's destroyed.
|
||||
*/
|
||||
public timed<T>(promise: Promise<T>, what: string, ms = nntpConfig.commandTimeoutMs): Promise<T> {
|
||||
let timer: NodeJS.Timeout
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`NNTP ${what}: no answer within ${ms / 1000} s`)), ms)
|
||||
})
|
||||
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer))
|
||||
}
|
||||
|
||||
/** Fetches a raw (still dot-stuffed) article body as a Buffer. */
|
||||
public async fetchBody(conn: any, messageId: string): Promise<Buffer> {
|
||||
const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
|
||||
const { data } = await conn.body(id)
|
||||
return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
|
||||
}
|
||||
|
||||
public async shutdown() {
|
||||
console.log('Shutting down all connections in the pool.');
|
||||
// A connection the server already closed may never answer QUIT.
|
||||
const quit = (conn: any) =>
|
||||
Promise.race([conn.quit(), new Promise((resolve) => setTimeout(resolve, 2000).unref())])
|
||||
.catch(() => {})
|
||||
.finally(() => this._destroy(conn));
|
||||
await Promise.all([...this.allConnections].map(quit));
|
||||
this.allConnections.clear();
|
||||
this.idleConnections.length = 0;
|
||||
this.waiters.length = 0;
|
||||
this.lowWaiters.length = 0;
|
||||
this.createdCount = 0;
|
||||
}
|
||||
}
|
||||
|
||||
export default new NntpService(nntpConfig.poolSize);
|
||||
@@ -0,0 +1,229 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { type ReleaseIdentity } from '#lib/release/index'
|
||||
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
||||
import { parseSubject } from '#lib/subject/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
import FileStore from '#services/FileStore'
|
||||
import { isArchiveStart } from '#lib/content/index'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
|
||||
export interface NzbImportResult {
|
||||
releaseId: number
|
||||
releaseName: string
|
||||
listed: number
|
||||
known: number
|
||||
added: number
|
||||
password: boolean
|
||||
}
|
||||
|
||||
/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
|
||||
export function nzbFileName(subject: string): string {
|
||||
const parsed = parseSubject(subject)
|
||||
const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1]
|
||||
if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
|
||||
return unquotedFileName(subject)
|
||||
}
|
||||
|
||||
/**
|
||||
* A filename from a subject without quotes, as some spot NZBs have:
|
||||
* `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`,
|
||||
* `06 - Song.flac Artist - Album (2020) (01/35)`. Counters, "yEnc" and a
|
||||
* trailing size are dropped; then the name ends at the first extension
|
||||
* followed by a space.
|
||||
*/
|
||||
export function unquotedFileName(subject: string): string {
|
||||
const cleaned = subject
|
||||
.replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ')
|
||||
.replace(/\s\d+\s*$/, ' ')
|
||||
.replace(/(\s+-)+\s*$/, '')
|
||||
.replace(/^\s*-\s+/, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned)
|
||||
const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned
|
||||
return name.replace(/\s+-$/, '').trim() || subject.trim()
|
||||
}
|
||||
|
||||
export interface NzbImportOptions {
|
||||
/**
|
||||
* For NZBs inside an archive: the entry's name. It keeps several NZBs of
|
||||
* one archive apart and names their release when the NZB has no name.
|
||||
*/
|
||||
entryName?: string
|
||||
/** Move the posted file into the release the NZB describes (default true). */
|
||||
moveContainer?: boolean
|
||||
/** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */
|
||||
name?: string
|
||||
/** Release source for new releases (default `nzb`). */
|
||||
source?: ReleaseIdentity['source']
|
||||
/**
|
||||
* The NZB is authoritative for which files form the release and what it is
|
||||
* called (Spotnet spots): its files get the NZB's own release, named
|
||||
* `name`, also when name rules had put them in another release. Files
|
||||
* another trusted NZB claimed first stay where they are.
|
||||
*/
|
||||
trusted?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26).
|
||||
* Files it lists that are already known (by first segment) keep their
|
||||
* release, which the NZB then joins; the others are added as files, so posts
|
||||
* spread over groups we do not index still become complete.
|
||||
*/
|
||||
class NzbImportService {
|
||||
async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise<NzbImportResult | null> {
|
||||
const nzb = parseNzb(xml)
|
||||
if (!nzb || nzb.files.length === 0) return null
|
||||
|
||||
const entries = nzb.files.map((f) => ({
|
||||
file: f,
|
||||
name: nzbFileName(f.subject),
|
||||
firstId: f.segments[0].id,
|
||||
}))
|
||||
|
||||
const added: { id: number; filename: string }[] = []
|
||||
const result = await db.transaction(async (trx) => {
|
||||
const known = await trx
|
||||
.from('files')
|
||||
.whereIn('first_message_id', entries.map((e) => e.firstId))
|
||||
.select('id', 'release_id', 'first_message_id')
|
||||
const knownIds = new Set(known.map((f) => f.first_message_id))
|
||||
|
||||
// Join the release the listed files are already in, else the posted NZB's own, else a new one.
|
||||
const counts = new Map<number, number>()
|
||||
for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
|
||||
let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
|
||||
const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
|
||||
const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}`
|
||||
const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
|
||||
|
||||
// Releases of other trusted NZBs: their files are not taken over.
|
||||
let claimed = new Set<number>()
|
||||
if (options.trusted) {
|
||||
// Only while that NZB still stands: a disposed spot's files go to its correction.
|
||||
const holders = await trx
|
||||
.from('releases as r')
|
||||
.whereIn('r.id', [...counts.keys()])
|
||||
.where('r.source', 'spot')
|
||||
.whereNot('r.release_key', key)
|
||||
.whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy']))
|
||||
.select('r.id')
|
||||
claimed = new Set(holders.map((r) => Number(r.id)))
|
||||
// Join another spot's release when it holds most of these files (a repost); else use our own.
|
||||
if (releaseId && !claimed.has(releaseId)) releaseId = undefined
|
||||
if (releaseId) claimed.delete(releaseId)
|
||||
}
|
||||
if (!releaseId) {
|
||||
const ids = await IngestService.upsertReleases(trx.knexClient, [
|
||||
{
|
||||
release: { key, name: title, source: options.source ?? 'nzb' },
|
||||
poster: nzbFile.poster,
|
||||
groups: nzbFile.groups ?? [],
|
||||
postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined,
|
||||
},
|
||||
])
|
||||
releaseId = ids.get(key)!
|
||||
}
|
||||
|
||||
const now = new Date()
|
||||
const missing = entries.filter((e) => !knownIds.has(e.firstId))
|
||||
for (let i = 0; i < missing.length; i += 500) {
|
||||
added.push(
|
||||
...(await FileStore.insert(
|
||||
missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
|
||||
file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
|
||||
filename: name,
|
||||
poster: file.poster || nzbFile.poster,
|
||||
date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
|
||||
parts: file.segments.length,
|
||||
size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
|
||||
segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })),
|
||||
groups: JSON.stringify(file.groups),
|
||||
release_id: releaseId,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
})),
|
||||
trx
|
||||
))
|
||||
)
|
||||
}
|
||||
// The poster's NZB says which files form the post: known files join this
|
||||
// release, also when name rules had split them over several.
|
||||
const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id)))
|
||||
const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))]
|
||||
await trx
|
||||
.from('files')
|
||||
.whereIn('id', moving.map((f) => f.id))
|
||||
.update({ release_id: releaseId })
|
||||
if (previous.length) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`,
|
||||
[releaseId, releaseId, releaseId]
|
||||
)
|
||||
await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete()
|
||||
}
|
||||
// The NZB file joins the release it describes, unless it is part of a real release already.
|
||||
const own = nzbFile.release_id ? Number(nzbFile.release_id) : null
|
||||
if (own !== releaseId && options.moveContainer !== false) {
|
||||
const ownRelease = own ? await trx.from('releases').where('id', own).first() : null
|
||||
if (!ownRelease || ownRelease.file_count <= 1) {
|
||||
await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId })
|
||||
if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete()
|
||||
}
|
||||
}
|
||||
|
||||
const release = await trx.from('releases').where('id', releaseId).first()
|
||||
const update: Record<string, unknown> = {}
|
||||
if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' })
|
||||
if (!release.file_total) update.file_total = entries.length
|
||||
if (options.trusted) {
|
||||
// Only its own release: a repost joining another spot's release keeps that one's title.
|
||||
if (release.release_key === key && release.name !== title) update.name = title
|
||||
if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source
|
||||
} else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) {
|
||||
update.name = title
|
||||
}
|
||||
if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
|
||||
|
||||
const result = {
|
||||
releaseId,
|
||||
releaseName: (update.name as string) ?? release.name,
|
||||
listed: entries.length,
|
||||
known: known.length,
|
||||
added: missing.length,
|
||||
password: Boolean(nzb.meta.password),
|
||||
}
|
||||
logger.info(
|
||||
`NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` +
|
||||
`${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.`
|
||||
)
|
||||
return result
|
||||
})
|
||||
// Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
|
||||
const archives = added.filter((f) => isArchiveStart(f.filename))
|
||||
if (archives.length) {
|
||||
await InspectionQueue.add(archives.map((f) => Number(f.id)))
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
|
||||
private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string {
|
||||
if (nzb.meta.name) return nzb.meta.name
|
||||
const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : ''
|
||||
if (entry && !looksRandom(entry)) return entry
|
||||
const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? ''))
|
||||
if (own && !looksRandom(own)) return own
|
||||
const largest = nzb.files
|
||||
.map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) }))
|
||||
.sort((a, b) => b.bytes - a.bytes)[0]
|
||||
return fileStem(largest?.name ?? own)
|
||||
}
|
||||
}
|
||||
|
||||
export default new NzbImportService()
|
||||
@@ -0,0 +1,133 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
|
||||
import type { Par2Set } from '#lib/par2/index'
|
||||
import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
|
||||
import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'
|
||||
|
||||
/** Size-only matching below this is too ambiguous. */
|
||||
const SIZE_ONLY_MIN = 1024 * 1024
|
||||
|
||||
function countBy<T>(values: T[]): Map<T, number> {
|
||||
const counts = new Map<T, number>()
|
||||
for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1)
|
||||
return counts
|
||||
}
|
||||
|
||||
export interface Par2Result {
|
||||
releaseId: number
|
||||
releaseName: string
|
||||
listed: number
|
||||
matched: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognises releases of obfuscated posts from their par2 files: the par2
|
||||
* lists the real name and exact size of every file in the set, and
|
||||
* body-derived binaries carry their exact size. See CONCEPT.md (D16).
|
||||
*/
|
||||
class Par2Service {
|
||||
async attach(binary: any, set: Par2Set): Promise<Par2Result> {
|
||||
const result = await db.transaction(async (trx) => {
|
||||
// Exact: size and first-16-KiB MD5 of a listed file. Only these may pull
|
||||
// the set into a release the files already belong to; size alone is
|
||||
// ambiguous across posts (standard volume sizes recur).
|
||||
const exact = await trx
|
||||
.from('binaries')
|
||||
.whereIn(['size', 'md5_16k'] as any, set.files.map((f) => [f.length, f.md5_16k]) as any)
|
||||
.select('id', 'release_id')
|
||||
|
||||
// Size only, for files without a fingerprint: the size must be listed
|
||||
// once in this set, be large, and belong to exactly one binary nearby.
|
||||
const listedOnce = [...countBy(set.files.map((f) => f.length))]
|
||||
.filter(([size, n]) => n === 1 && size >= SIZE_ONLY_MIN)
|
||||
.map(([size]) => size)
|
||||
const bySize = listedOnce.length
|
||||
? await trx
|
||||
.from('binaries')
|
||||
.whereIn('size', listedOnce)
|
||||
.whereNull('md5_16k')
|
||||
.whereRaw('abs(extract(epoch FROM (posted_at - ?::timestamptz))) < ?', [
|
||||
binary.posted_at,
|
||||
PAR2_MATCH_WINDOW_SECONDS,
|
||||
])
|
||||
.select('id', 'release_id', 'size')
|
||||
: []
|
||||
const sizeCounts = countBy(bySize.map((b) => Number(b.size)))
|
||||
const fuzzy = bySize.filter((b) => sizeCounts.get(Number(b.size)) === 1)
|
||||
const candidates = [...exact, ...fuzzy]
|
||||
|
||||
// Prefer a release the exactly matched files are already in (e.g.
|
||||
// obfuscated volumes grouped by their stem); otherwise the set's own.
|
||||
const counts = countBy(exact.filter((c) => c.release_id).map((c) => Number(c.release_id)))
|
||||
let releaseId = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
|
||||
const identity = par2ReleaseIdentity(
|
||||
set.recoverySetId,
|
||||
set.files.map((f) => f.name)
|
||||
)
|
||||
if (!releaseId) {
|
||||
const ids = await IngestService.upsertReleases(trx.knexClient, [
|
||||
{
|
||||
release: identity,
|
||||
poster: binary.poster,
|
||||
weakIdentity: true,
|
||||
groups: binary.groups,
|
||||
postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
|
||||
},
|
||||
])
|
||||
releaseId = ids.get(identity.key)!
|
||||
}
|
||||
|
||||
await trx
|
||||
.knexClient('release_files')
|
||||
.insert(
|
||||
set.files.map((f) => ({
|
||||
release_id: releaseId,
|
||||
par2_file_id: f.fileId,
|
||||
name: f.name,
|
||||
size: f.length,
|
||||
md5_16k: f.md5_16k,
|
||||
md5: f.md5,
|
||||
set_id: set.recoverySetId,
|
||||
}))
|
||||
)
|
||||
.onConflict(['release_id', 'par2_file_id'])
|
||||
.ignore()
|
||||
|
||||
const matchIds = candidates.filter((c) => !c.release_id).map((c) => Number(c.id))
|
||||
await trx.from('binaries').whereIn('id', [binary.id, ...matchIds]).whereNull('release_id').update({ release_id: releaseId })
|
||||
// Exact matches sitting alone in a random-named release (one file of a
|
||||
// season, each posted under its own random name) join the set's release.
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET release_id = ? WHERE b.id = ANY(?) AND b.release_id <> ? AND ${WEAK_RELEASE_SQL('b.release_id')}`,
|
||||
[releaseId, exact.map((c) => Number(c.id)), releaseId]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE files f SET release_id = b.release_id FROM binaries b
|
||||
WHERE b.file_id = f.id AND b.release_id = ? AND f.release_id IS DISTINCT FROM b.release_id`,
|
||||
[releaseId]
|
||||
)
|
||||
|
||||
// The set may have joined a release named after one random filename.
|
||||
const release = await trx.from('releases').where('id', releaseId).first()
|
||||
let releaseName = release?.name
|
||||
const better = par2ReleaseName(set.files.map((f) => f.name))
|
||||
// A spot's title is kept (CONCEPT.md D29).
|
||||
if (releaseName && better && release.source !== 'spot' && isBetterName(releaseName, better)) {
|
||||
await trx.from('releases').where('id', releaseId).update({ name: better })
|
||||
releaseName = better
|
||||
}
|
||||
logger.info(
|
||||
`Par2 set ${set.recoverySetId.slice(0, 8)} → release ${releaseId} "${releaseName}": ` +
|
||||
`${set.files.length} files listed, ${matchIds.length} binaries matched by size.`
|
||||
)
|
||||
return { releaseId, releaseName, listed: set.files.length, matched: matchIds.length }
|
||||
})
|
||||
|
||||
// Real names for every known file of the set (exact fingerprint, or size within the release).
|
||||
await NameService.resolveForRelease(result.releaseId)
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
export default new Par2Service()
|
||||
@@ -0,0 +1,65 @@
|
||||
import { Queue, type DefaultJobOptions } from 'bullmq'
|
||||
import queueConfig from '#config/queue'
|
||||
import gcConfig from '#config/gc'
|
||||
|
||||
const defaultJobOptions: DefaultJobOptions = {
|
||||
// Transient NNTP errors (e.g. a pooled connection the server closed) are retried.
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
removeOnComplete: {
|
||||
age: gcConfig.jobs.completedAgeSeconds,
|
||||
count: gcConfig.jobs.completedMaxCount,
|
||||
},
|
||||
removeOnFail: { age: gcConfig.jobs.failedAgeSeconds },
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues are created (and connect to Redis) on first use. Ace imports every
|
||||
* command at boot, so eager queues would keep one-off commands from exiting.
|
||||
*/
|
||||
class QueueService {
|
||||
private readonly queues = new Map<string, Queue>()
|
||||
|
||||
private queue(name: string): Queue {
|
||||
let queue = this.queues.get(name)
|
||||
if (!queue) {
|
||||
queue = new Queue(name, { connection: queueConfig.connection, defaultJobOptions })
|
||||
this.queues.set(name, queue)
|
||||
}
|
||||
return queue
|
||||
}
|
||||
|
||||
get nntpFetchQueue() {
|
||||
return this.queue('nntp-fetch-queue')
|
||||
}
|
||||
get fileQueue() {
|
||||
return this.queue('file-queue')
|
||||
}
|
||||
get bodyQueue() {
|
||||
return this.queue('body-queue')
|
||||
}
|
||||
get collectionQueue() {
|
||||
return this.queue('collection-queue')
|
||||
}
|
||||
/** Complete archive listings, at low priority (ArchiveWalkService). */
|
||||
get archiveWalkQueue() {
|
||||
return this.queue('archive-walk-queue')
|
||||
}
|
||||
|
||||
/** Queues no longer used by the pipeline, kept here so GC can empty them. */
|
||||
get legacy(): Queue[] {
|
||||
return [this.queue('header-queue')]
|
||||
}
|
||||
|
||||
get all(): Queue[] {
|
||||
return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue, this.archiveWalkQueue]
|
||||
}
|
||||
|
||||
/** Closes the queues that were opened. */
|
||||
async closeAll() {
|
||||
await Promise.all([...this.queues.values()].map((queue) => queue.close()))
|
||||
this.queues.clear()
|
||||
}
|
||||
}
|
||||
|
||||
export default new QueueService()
|
||||
@@ -0,0 +1,25 @@
|
||||
import { createRequire } from 'module'
|
||||
import redisConfig from '#config/redis'
|
||||
|
||||
const require = createRequire(import.meta.url)
|
||||
const IORedis = require('ioredis')
|
||||
|
||||
/**
|
||||
* Connects on first use. Ace imports every command at boot, so an eager
|
||||
* connection would keep even `node ace list` from exiting.
|
||||
*/
|
||||
class RedisService {
|
||||
private instance?: any
|
||||
|
||||
public get client(): any {
|
||||
this.instance ??= new IORedis(redisConfig.connections.main)
|
||||
return this.instance
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.instance) await this.instance.quit()
|
||||
this.instance = undefined
|
||||
}
|
||||
}
|
||||
|
||||
export default new RedisService()
|
||||
@@ -0,0 +1,35 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Noted releases taken per transaction. */
|
||||
const BATCH = 2000
|
||||
|
||||
/**
|
||||
* Release completeness (D24) for releases whose files or par2 listing
|
||||
* changed: triggers only note them in release_dirty (a release can be noted
|
||||
* thousands of times between sweeps), and gc:scheduler recomputes each
|
||||
* noted release once every RELEASE_COMPLETE_INTERVAL_MS (LOG.md 2026-10-06 (6)).
|
||||
*/
|
||||
class ReleaseCompleteness {
|
||||
/** Recomputes all noted releases; returns how many distinct releases were checked. */
|
||||
async sweep(): Promise<number> {
|
||||
let checked = 0
|
||||
for (;;) {
|
||||
const done = await db.transaction(async (trx) => {
|
||||
// Taken and recomputed in one transaction: a failure leaves the notes for the next sweep.
|
||||
const { rows } = await trx.rawQuery(
|
||||
`DELETE FROM release_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM release_dirty LIMIT ?)) RETURNING release_id`,
|
||||
[BATCH]
|
||||
)
|
||||
const ids: number[] = [...new Set<number>(rows.map((row: { release_id: string }) => Number(row.release_id)))]
|
||||
if (ids.length > 0) {
|
||||
await trx.rawQuery(`SELECT recompute_release_complete(id) FROM unnest(?::bigint[]) AS id`, [ids])
|
||||
}
|
||||
checked += ids.length
|
||||
return rows.length < BATCH
|
||||
})
|
||||
if (done) return checked
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new ReleaseCompleteness()
|
||||
@@ -0,0 +1,42 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/**
|
||||
* Uploads split per poster (CONCEPT.md D35). Some uploaders post every
|
||||
* volume of a RAR set under a different poster; the post-id grouping then
|
||||
* makes one release per volume, all with the same name
|
||||
* (`Masters.of.the.Universe.1987…`: 191 one-file releases of one 199-volume
|
||||
* set). Such releases are one upload when all their files share one stem and
|
||||
* no file name repeats (a repost would repeat them, another upload would
|
||||
* bring another stem); their files move into the oldest of them, and GC
|
||||
* deletes the emptied ones.
|
||||
*/
|
||||
class ReleaseMergeService {
|
||||
async mergeSplitPosts({ dryRun = false } = {}): Promise<{ names: number; releases: number }> {
|
||||
const { rows } = await db.rawQuery(`
|
||||
WITH dup AS (
|
||||
SELECT name FROM releases
|
||||
WHERE file_count > 0 AND source = 'post' AND is_readable_name(name)
|
||||
GROUP BY name HAVING count(*) > 1
|
||||
)
|
||||
SELECT r.name, array_agg(DISTINCT r.id ORDER BY r.id) AS ids
|
||||
FROM dup JOIN releases r ON r.name = dup.name AND r.file_count > 0 AND r.source = 'post'
|
||||
JOIN files f ON f.release_id = r.id
|
||||
GROUP BY r.name
|
||||
HAVING count(DISTINCT name_stem(coalesce(f.real_name, f.filename))) = 1
|
||||
AND count(f.id) = count(DISTINCT coalesce(f.real_name, f.filename))
|
||||
AND max(r.posted_at) - min(r.posted_at) < interval '30 days'`)
|
||||
let releases = 0
|
||||
for (const { name, ids } of rows as { name: string; ids: (string | number)[] }[]) {
|
||||
const [target, ...others] = ids.map(Number)
|
||||
releases += others.length
|
||||
if (dryRun) continue
|
||||
await db.from('files').whereIn('release_id', others).update({ release_id: target })
|
||||
await db.from('binaries').whereIn('release_id', others).update({ release_id: target })
|
||||
logger.info(`Merged ${others.length + 1} releases of one upload split per poster into ${target} "${name}".`)
|
||||
}
|
||||
return { names: rows.length, releases }
|
||||
}
|
||||
}
|
||||
|
||||
export default new ReleaseMergeService()
|
||||
@@ -0,0 +1,168 @@
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { Job, Worker } from 'bullmq'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import queueConfig from '#config/queue'
|
||||
import nntpConfig from '#config/nntp'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NntpService from '#services/NntpService'
|
||||
import ContentService, { describeContent } from '#services/ContentService'
|
||||
import ArchiveWalkService from '#services/ArchiveWalkService'
|
||||
import { isMissingArticle, type ContentTask, type InspectProbe, type InspectTask, type WalkProbe } from '#lib/content/probe'
|
||||
|
||||
/** Inspection first; archive walks (D40) only when it has nothing waiting. */
|
||||
const QUEUES = ['collection-queue', 'archive-walk-queue'] as const
|
||||
type QueueName = (typeof QUEUES)[number]
|
||||
|
||||
/** A node has this long to post a job's result; after that the job goes back to the queue (BullMQ's stalled check). */
|
||||
const LOCK_MS = 10 * 60 * 1000
|
||||
|
||||
export interface RemoteJob {
|
||||
queue: QueueName
|
||||
jobId: string
|
||||
/** The job's lock: whoever holds it may complete the job. */
|
||||
token: string
|
||||
task: ContentTask
|
||||
}
|
||||
|
||||
export class JobLost extends Error {}
|
||||
export class ArticleRelayOff extends Error {}
|
||||
|
||||
/**
|
||||
* Content inspection and archive walks on seeding nodes (CONCEPT.md D41).
|
||||
* Nodes take jobs from the same queues worker:collection works on: a lease
|
||||
* locks a job (BullMQ's manual processing) and gives the node a task that
|
||||
* was prepared from the database; the node probes it, fetching articles
|
||||
* through the relay (articles()), and posts the result, which is applied
|
||||
* here. A job whose node goes quiet is unlocked after LOCK_MS and picked up
|
||||
* again by whoever comes next.
|
||||
*/
|
||||
class RemoteContentService {
|
||||
private readonly workers = new Map<QueueName, Worker>()
|
||||
|
||||
/** Up to `max` jobs for a node, each locked for LOCK_MS. */
|
||||
async lease(nodeId: number, max: number): Promise<RemoteJob[]> {
|
||||
const jobs: RemoteJob[] = []
|
||||
for (const queue of QUEUES) {
|
||||
while (jobs.length < max) {
|
||||
const token = `node-${nodeId}:${randomUUID()}`
|
||||
const job = await this.worker(queue).getNextJob(token, { block: false })
|
||||
if (!job) break
|
||||
try {
|
||||
const task = await this.prepare(queue, job)
|
||||
if (!task) {
|
||||
await job.moveToCompleted(null, token, false)
|
||||
continue
|
||||
}
|
||||
// What applying needs, kept with the job: the node's copy of the task isn't trusted.
|
||||
if (task.kind === 'inspect') await job.updateData({ ...job.data, remote: { fileId: task.fileId, name: task.name } })
|
||||
jobs.push({ queue, jobId: job.id!, token, task })
|
||||
} catch (error: any) {
|
||||
await job.moveToFailed(error, token, false).catch(() => {})
|
||||
}
|
||||
}
|
||||
}
|
||||
return jobs
|
||||
}
|
||||
|
||||
/** Applies a node's result and completes the job. */
|
||||
async finish(queue: QueueName, jobId: string, token: string, result: InspectProbe | WalkProbe) {
|
||||
const job = await this.job(queue, jobId)
|
||||
let summary: unknown
|
||||
if (queue === 'collection-queue') {
|
||||
const remote = job.data.remote
|
||||
if (!remote) throw new JobLost(`Job ${jobId} was not leased to a node`)
|
||||
const task: InspectTask = { kind: 'inspect', fileId: Number(remote.fileId), name: String(remote.name), segments: [], sevenZip: null }
|
||||
const content = await ContentService.apply(task, result as InspectProbe)
|
||||
for (const line of describeContent(task.name, content)) logger.info(`[node] ${line}`)
|
||||
summary = content.contentType
|
||||
} else {
|
||||
const outcome = await ArchiveWalkService.apply(Number(job.data.releaseId), result as WalkProbe)
|
||||
logger.info(
|
||||
`[node] Archive walk of release ${outcome.releaseId}: ${outcome.before} → ${outcome.entries} entries from ${outcome.segments} segments` +
|
||||
(outcome.complete ? '' : ` (stopped: ${outcome.stopped})`) +
|
||||
(outcome.renamed ? `, renamed to "${outcome.renamed}"` : '')
|
||||
)
|
||||
summary = outcome
|
||||
}
|
||||
await this.locked(() => job.moveToCompleted(summary, token, false))
|
||||
}
|
||||
|
||||
/** The probe failed on the node (e.g. a missing article): retried like a local failure. */
|
||||
async fail(queue: QueueName, jobId: string, token: string, error: string) {
|
||||
const job = await this.job(queue, jobId)
|
||||
// A missing first article from a node that predates the probe's own handling: a result,
|
||||
// not something to retry (each try costs Eweka ~450 ms; LOG.md 2026-10-06 (14)).
|
||||
if (queue === 'collection-queue' && job.data.remote && isMissingArticle(error)) {
|
||||
return this.finish(queue, jobId, token, { size: 0, contentType: 'missing', md5_16k: null, yencName: '' })
|
||||
}
|
||||
await this.locked(() => job.moveToFailed(new Error(`node: ${error}`), token, false))
|
||||
}
|
||||
|
||||
/** A stopping node gives its jobs back, without using up an attempt. */
|
||||
async release(queue: QueueName, jobId: string, token: string) {
|
||||
const job = await this.job(queue, jobId)
|
||||
await this.locked(() => job.moveToWait(token))
|
||||
}
|
||||
|
||||
/** One article body for a relay node, on this process's NNTP pool. Null when the server doesn't have it. */
|
||||
async article(messageId: string): Promise<Buffer | null> {
|
||||
// NNTP_POOL_SIZE 0 (SEED_ARTICLE_CONNECTIONS=0): no connections for this; acquire() would wait forever.
|
||||
if (nntpConfig.poolSize <= 0) throw new ArticleRelayOff('Article relay is off (SEED_ARTICLE_CONNECTIONS=0)')
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
|
||||
} catch (error: any) {
|
||||
// 430 no such article: a fact about the article, the connection is fine.
|
||||
if (/^4\d\d/.test(String(error.message))) return null
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes the lease workers' Redis connections (tests; a server just exits). */
|
||||
async close() {
|
||||
await Promise.all([...this.workers.values()].map((worker) => worker.close()))
|
||||
this.workers.clear()
|
||||
}
|
||||
|
||||
isQueue(name: unknown): name is QueueName {
|
||||
return QUEUES.includes(name as QueueName)
|
||||
}
|
||||
|
||||
private async prepare(queue: QueueName, job: Job): Promise<ContentTask | null> {
|
||||
if (queue === 'collection-queue') return ContentService.prepare(Number(job.data.fileId))
|
||||
const prepared = await ArchiveWalkService.prepare(Number(job.data.releaseId))
|
||||
return 'task' in prepared ? prepared.task : null
|
||||
}
|
||||
|
||||
private async job(queue: QueueName, jobId: string): Promise<Job> {
|
||||
const job = await Job.fromId(queue === 'collection-queue' ? QueueService.collectionQueue : QueueService.archiveWalkQueue, jobId)
|
||||
if (!job) throw new JobLost(`Job ${jobId} is gone`)
|
||||
return job
|
||||
}
|
||||
|
||||
/** BullMQ refuses a job whose lock another holder has now: the node's result came too late. */
|
||||
private async locked(action: () => Promise<unknown>) {
|
||||
try {
|
||||
await action()
|
||||
} catch (error: any) {
|
||||
if (/lock|not in the active state|Missing key/i.test(String(error.message))) throw new JobLost(error.message)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/** A worker that only hands out jobs (no processor, never runs). */
|
||||
private worker(queue: QueueName): Worker {
|
||||
let worker = this.workers.get(queue)
|
||||
if (!worker) {
|
||||
worker = new Worker(queue, null, { connection: queueConfig.connection, lockDuration: LOCK_MS, autorun: false })
|
||||
this.workers.set(queue, worker)
|
||||
}
|
||||
return worker
|
||||
}
|
||||
}
|
||||
|
||||
export default new RemoteContentService()
|
||||
@@ -0,0 +1,421 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { buildNzb } from '#lib/nzb/index'
|
||||
import SpotService from '#services/SpotService'
|
||||
import { segmentsOf } from '#services/FileStore'
|
||||
import { spotCategoryLabels } from '#lib/spotnet/index'
|
||||
|
||||
/**
|
||||
* `known`: we have the password (it's in our NZB, so SABnzbd/NZBGet unpack
|
||||
* without asking); `missing`: the archive is encrypted and we don't have
|
||||
* it; `none`: no password needed as far as we know.
|
||||
*/
|
||||
export type PasswordState = 'known' | 'missing' | 'none'
|
||||
|
||||
export function passwordState(encrypted: boolean | null, hasPassword: boolean): PasswordState {
|
||||
if (hasPassword) return 'known'
|
||||
return encrypted ? 'missing' : 'none'
|
||||
}
|
||||
|
||||
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
||||
|
||||
export interface SearchParams {
|
||||
q?: string
|
||||
/** Only releases whose name is readable (not an obfuscated token). */
|
||||
readable?: boolean
|
||||
/** Only releases that have all their files. */
|
||||
complete?: boolean
|
||||
/** Hide releases that need a password we don't have. */
|
||||
hideEncrypted?: boolean
|
||||
source?: string
|
||||
poster?: string
|
||||
sort?: SortOrder
|
||||
page?: number
|
||||
perPage?: number
|
||||
/** Newznab: exact category ids (already expanded from parents). */
|
||||
categories?: number[]
|
||||
/** Newznab: leave out releases known to be missing files. */
|
||||
notIncomplete?: boolean
|
||||
maxAgeDays?: number
|
||||
/** TV: season and episode numbers, matched as S01E02 or 1x02 in the name. */
|
||||
season?: number
|
||||
episode?: number
|
||||
/** Overrides `page` (Newznab uses offsets). */
|
||||
offset?: number
|
||||
/** Leave out releases that are only an encrypted container (nzbe). */
|
||||
hideContainers?: boolean
|
||||
/** Include incomplete releases that consist of par2 files only (hidden by default). */
|
||||
includePar2Only?: boolean
|
||||
/** Size bounds in bytes (releases without a known size never match). */
|
||||
minSize?: number
|
||||
maxSize?: number
|
||||
/** File extensions (without the dot); a release matches if a file or archive entry has one of them (`releases.extensions`). */
|
||||
extensions?: string[]
|
||||
/** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
|
||||
countLimit?: number
|
||||
}
|
||||
|
||||
const SORTS: Record<SortOrder, string> = {
|
||||
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
|
||||
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
|
||||
largest: 'r.size DESC, r.id DESC',
|
||||
files: 'r.file_count DESC, r.id DESC',
|
||||
name: 'r.name ASC, r.id ASC',
|
||||
}
|
||||
|
||||
/**
|
||||
* Shown in search: releases with files, and Spotnet listings whose NZB is
|
||||
* fetched on first download (CONCEPT.md D29). Matches the partial indexes.
|
||||
*/
|
||||
const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')`
|
||||
|
||||
/** Incomplete, and every file is a par2 file: nothing to download but repair data. */
|
||||
const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS (
|
||||
SELECT 1 FROM files f LEFT JOIN file_details d ON d.file_id = f.id WHERE f.release_id = r.id
|
||||
AND d.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
|
||||
|
||||
/** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */
|
||||
function pattern(word: string): string {
|
||||
const escaped = word.replace(/[\\%]/g, (c) => `\\${c}`)
|
||||
return `%${escaped.replace(/[._\- ]+/g, '_')}%`
|
||||
}
|
||||
|
||||
/**
|
||||
* As SQL's search_normalize(): lowercase, and dots, dashes, underscores and
|
||||
* spaces as one space. `The.Block.AU` and `the block au` search the same.
|
||||
*/
|
||||
export function searchNormalize(text: string): string {
|
||||
return text.toLowerCase().replace(/[._\s-]+/g, ' ').trim()
|
||||
}
|
||||
|
||||
/** How long the web UI's totals are reused. */
|
||||
const STATS_TTL_MS = 60 * 1000
|
||||
|
||||
/**
|
||||
* Counting stops here: a broad search needn't count every match. Each
|
||||
* counted row is a random heap read, so the web UI counts less (it only
|
||||
* shows "1,000+"); Newznab clients page by this total and keep the default.
|
||||
*/
|
||||
const COUNT_LIMIT = 10_000
|
||||
|
||||
/**
|
||||
* Searching and browsing releases for the web interface (CONCEPT.md D22).
|
||||
*/
|
||||
class SearchService {
|
||||
async search(params: SearchParams) {
|
||||
const where: string[] = [
|
||||
VISIBLE_SQL,
|
||||
// Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
|
||||
`NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status IN ('removed', 'dummy'))`,
|
||||
]
|
||||
const bindings: unknown[] = []
|
||||
|
||||
// Every word must occur in the release's search text: its name, archive
|
||||
// listing and file names (CONCEPT.md D33), one trigram index for all.
|
||||
const words = (params.q ?? '').trim().split(/\s+/).map(searchNormalize).filter(Boolean).slice(0, 8)
|
||||
for (const word of words) {
|
||||
where.push(`r.search_text LIKE ?`)
|
||||
bindings.push(`%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`)
|
||||
}
|
||||
if (params.readable) where.push('is_readable_name(r.name)')
|
||||
if (params.minSize !== undefined) {
|
||||
where.push('r.size >= ?')
|
||||
bindings.push(params.minSize)
|
||||
}
|
||||
if (params.maxSize !== undefined) {
|
||||
where.push('r.size <= ?')
|
||||
bindings.push(params.maxSize)
|
||||
}
|
||||
// File types from file names and archive listings (migration …130), via their GIN index.
|
||||
const extensions = (params.extensions ?? []).map((e) => e.toLowerCase().replace(/^\./, '')).filter((e) => /^[a-z0-9]{1,8}$/.test(e)).slice(0, 8)
|
||||
if (extensions.length > 0) {
|
||||
where.push('r.extensions && ?::text[]')
|
||||
bindings.push(extensions)
|
||||
}
|
||||
if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
|
||||
if (params.hideContainers) where.push('r.container IS NULL')
|
||||
// par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
|
||||
if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
|
||||
// Kept current by triggers (migration …860).
|
||||
if (params.complete) where.push('r.complete')
|
||||
if (params.source) {
|
||||
where.push('r.source = ?')
|
||||
bindings.push(params.source)
|
||||
}
|
||||
if (params.poster) {
|
||||
where.push('r.poster ILIKE ?')
|
||||
bindings.push(pattern(params.poster))
|
||||
}
|
||||
if (params.categories?.length) {
|
||||
where.push('r.category = ANY(?)')
|
||||
bindings.push(params.categories)
|
||||
}
|
||||
if (params.notIncomplete) {
|
||||
where.push(`(r.complete OR (r.file_total IS NULL AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)))`)
|
||||
}
|
||||
if (params.maxAgeDays) {
|
||||
where.push(`r.posted_at >= now() - make_interval(days => ?)`)
|
||||
bindings.push(params.maxAgeDays)
|
||||
}
|
||||
if (params.season !== undefined) {
|
||||
const s = params.season
|
||||
const e = params.episode
|
||||
where.push('r.name ~* ?')
|
||||
bindings.push(
|
||||
e !== undefined
|
||||
? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)`
|
||||
: `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`
|
||||
)
|
||||
}
|
||||
|
||||
const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
|
||||
const page = Math.max(params.page ?? 1, 1)
|
||||
const offset = params.offset ?? (page - 1) * perPage
|
||||
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
||||
const whereSql = where.join(' AND ')
|
||||
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
||||
|
||||
const [{ rows }, { rows: countRows }] = await Promise.all([
|
||||
db.rawQuery(
|
||||
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.file_count, r.category, r.container,
|
||||
r.size, r.encrypted, is_readable_name(r.name) AS readable,
|
||||
(SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id) AS listed_files,
|
||||
(SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id AND EXISTS (
|
||||
SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
|
||||
jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
|
||||
r.complete, ${PAR2_ONLY_SQL} AS par2_only, r.password IS NOT NULL AS has_password,
|
||||
(SELECT s.description FROM spots s WHERE s.release_id = r.id AND s.description IS NOT NULL LIMIT 1) AS spot_description
|
||||
FROM releases r WHERE ${whereSql}
|
||||
ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
[...bindings, perPage, offset]
|
||||
),
|
||||
db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
|
||||
])
|
||||
|
||||
return {
|
||||
page,
|
||||
perPage,
|
||||
offset,
|
||||
total: Math.min(Number(countRows[0].total), countLimit),
|
||||
/** More than `total` match (the count stops at `countLimit`). */
|
||||
totalCapped: Number(countRows[0].total) > countLimit,
|
||||
releases: rows.map((r: any) => ({
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
source: r.source,
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
postedAt: r.posted_at,
|
||||
fileTotal: r.file_total,
|
||||
fileCount: r.file_count,
|
||||
size: Number(r.size),
|
||||
encrypted: r.encrypted,
|
||||
readable: r.readable,
|
||||
listedFiles: Number(r.listed_files),
|
||||
listedPresent: Number(r.listed_present),
|
||||
archiveEntries: Number(r.archive_entries),
|
||||
complete: r.complete,
|
||||
category: r.category,
|
||||
container: r.container,
|
||||
par2Only: r.par2_only,
|
||||
listingOnly: r.source === 'spot' && Number(r.file_count) === 0,
|
||||
hasPassword: r.has_password,
|
||||
passwordState: passwordState(r.encrypted, r.has_password),
|
||||
spotDescription: r.spot_description,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
async release(id: number) {
|
||||
const release = await db.from('releases').where('id', id).first()
|
||||
if (!release) return null
|
||||
|
||||
const [files, listed, pending] = await Promise.all([
|
||||
db
|
||||
.from('files as f')
|
||||
.leftJoin('binaries as b', 'b.file_id', 'f.id')
|
||||
.where('f.release_id', id)
|
||||
.orderByRaw('coalesce(f.real_name, f.filename)')
|
||||
.select('f.id', 'f.filename', 'f.real_name', 'f.size', 'f.parts', 'f.date', 'f.poster', 'b.size as exact_size'),
|
||||
db.rawQuery(
|
||||
`SELECT rf.name, rf.size,
|
||||
EXISTS (SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = rf.release_id AND (f.real_name = rf.name OR b.size = rf.size)) AS present
|
||||
FROM release_files rf WHERE rf.release_id = ? ORDER BY rf.name`,
|
||||
[id]
|
||||
),
|
||||
db.rawQuery(
|
||||
`SELECT count(*) AS binaries, coalesce(sum(b.total_parts), 0) AS parts_total,
|
||||
coalesce(sum((SELECT count(*) FROM parts p WHERE p.binary_id = b.id)), 0) AS parts_have
|
||||
FROM binaries b WHERE b.release_id = ? AND b.completed_at IS NULL AND b.conflicted_at IS NULL`,
|
||||
[id]
|
||||
),
|
||||
])
|
||||
|
||||
const spotRow = await db.from('spots').where('release_id', id).whereNotIn('status', ['removed', 'dummy']).orderBy('posted_at', 'desc').first()
|
||||
const spot = spotRow ? await SpotService.withDetails(spotRow) : null
|
||||
return {
|
||||
id: Number(release.id),
|
||||
name: release.name,
|
||||
source: release.source,
|
||||
spot: spot
|
||||
? {
|
||||
messageId: spot.message_id,
|
||||
title: spot.title,
|
||||
tag: spot.tag,
|
||||
poster: spot.poster,
|
||||
postedAt: spot.posted_at,
|
||||
website: spot.website,
|
||||
description: spot.description,
|
||||
image: spot.image?.segments
|
||||
? { url: `/api/v1/releases/${id}/image`, width: spot.image.width ?? null, height: spot.image.height ?? null }
|
||||
: spot.image?.url
|
||||
? { external: spot.image.url }
|
||||
: null,
|
||||
categories: spotCategoryLabels(spot.category, spot.subcats),
|
||||
}
|
||||
: null,
|
||||
password: release.password ?? null,
|
||||
passwordSource: release.password_source ?? null,
|
||||
passwordState: passwordState(release.encrypted, release.password !== null),
|
||||
listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
|
||||
poster: release.poster,
|
||||
groups: release.groups,
|
||||
postedAt: release.posted_at,
|
||||
fileTotal: release.file_total,
|
||||
fileCount: release.file_count,
|
||||
size: Number(release.size),
|
||||
encrypted: release.encrypted,
|
||||
container: release.container,
|
||||
contents: release.contents ?? [],
|
||||
files: files.map((f) => ({
|
||||
id: Number(f.id),
|
||||
name: f.real_name ?? f.filename,
|
||||
postedName: f.filename,
|
||||
realName: f.real_name,
|
||||
size: Number(f.exact_size ?? f.size),
|
||||
parts: f.parts,
|
||||
postedAt: f.date ? new Date(Number(f.date)) : null,
|
||||
poster: f.poster,
|
||||
})),
|
||||
par2Files: listed.rows.map((r: any) => ({ name: r.name, size: Number(r.size), present: r.present })),
|
||||
pending: {
|
||||
binaries: Number(pending.rows[0].binaries),
|
||||
partsHave: Number(pending.rows[0].parts_have),
|
||||
partsTotal: Number(pending.rows[0].parts_total),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** The image of the spot behind a release: bytes and a media type sniffed from them, or null. */
|
||||
async spotImage(id: number): Promise<{ data: Buffer; type: string } | null> {
|
||||
const spot = await db.from('spots').where('release_id', id).whereNotNull('image').orderBy('posted_at', 'desc').first()
|
||||
if (!spot) return null
|
||||
const data = await SpotService.image(spot)
|
||||
if (!data) return null
|
||||
const type =
|
||||
data[0] === 0xff && data[1] === 0xd8 ? 'image/jpeg'
|
||||
: data.subarray(0, 4).toString('latin1') === '\x89PNG' ? 'image/png'
|
||||
: data.subarray(0, 3).toString('latin1') === 'GIF' ? 'image/gif'
|
||||
: data.subarray(0, 2).toString('latin1') === 'BM' ? 'image/bmp'
|
||||
: data.subarray(8, 12).toString('latin1') === 'WEBP' ? 'image/webp'
|
||||
: null
|
||||
return type ? { data, type } : null
|
||||
}
|
||||
|
||||
async nzb(id: number): Promise<{ name: string; xml: string } | null> {
|
||||
let release = await db.from('releases').where('id', id).first()
|
||||
if (!release) return null
|
||||
if (release.source === 'spot' && Number(release.file_count) === 0) {
|
||||
// A Spotnet listing: read its NZB from Usenet now (the import may join known files elsewhere).
|
||||
const imported = await SpotService.importListing(id)
|
||||
if (!imported) return null
|
||||
id = imported
|
||||
release = await db.from('releases').where('id', id).first()
|
||||
}
|
||||
const files = await db
|
||||
.from('files')
|
||||
.where('release_id', id)
|
||||
.orderByRaw('coalesce(real_name, filename)')
|
||||
.select('filename', 'real_name', 'poster', 'date', 'groups', 'segments')
|
||||
const xml = buildNzb(
|
||||
files.map((f) => ({
|
||||
name: f.real_name ?? f.filename,
|
||||
poster: f.poster,
|
||||
date: Number(f.date),
|
||||
groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
|
||||
segments: segmentsOf(f),
|
||||
})),
|
||||
{ title: release.name, password: release.password ?? undefined }
|
||||
)
|
||||
return { name: release.name, xml }
|
||||
}
|
||||
|
||||
/**
|
||||
* Totals for the web UI header. They count millions of rows (minutes with
|
||||
* the backfill running), so they are computed at most once per
|
||||
* STATS_TTL_MS and never twice at a time; callers in between share it.
|
||||
*/
|
||||
async stats() {
|
||||
if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
|
||||
this.statsRefresh ??= this.computeStats()
|
||||
.then((value) => {
|
||||
this.statsCache = { at: Date.now(), value }
|
||||
})
|
||||
// A failed background refresh keeps the previous totals; only a caller that waits sees the error.
|
||||
.catch((error) => {
|
||||
if (!this.statsCache) throw error
|
||||
})
|
||||
.finally(() => {
|
||||
this.statsRefresh = undefined
|
||||
})
|
||||
// Only the very first caller waits; later ones get the previous totals meanwhile.
|
||||
if (!this.statsCache) await this.statsRefresh
|
||||
}
|
||||
return this.statsCache!.value
|
||||
}
|
||||
|
||||
private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
|
||||
private statsRefresh?: Promise<void>
|
||||
|
||||
/**
|
||||
* Estimates, in milliseconds instead of minutes (CONCEPT.md D33): counts
|
||||
* from Postgres's statistics (visible releases = the entries of an index
|
||||
* that covers exactly them), ratios and sizes from a 0.5% sample of the
|
||||
* releases table. Within a few percent once autovacuum has analysed.
|
||||
*/
|
||||
private async computeStats() {
|
||||
const { rows } = await db.rawQuery(`
|
||||
WITH sample AS (
|
||||
SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
|
||||
FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
|
||||
SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
|
||||
-- files is partitioned (D39): its partitions hold the counts.
|
||||
(SELECT sum(c.reltuples) FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass) AS files,
|
||||
(SELECT count(*) FROM sample) AS sampled,
|
||||
(SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
|
||||
-- Indexed data only: a spot listing's size is what the spot announces.
|
||||
(SELECT coalesce(sum(size), 0) FROM sample WHERE file_count > 0) AS sampled_size,
|
||||
(SELECT count(*) FROM sample WHERE encrypted) AS sampled_encrypted,
|
||||
(SELECT max(updated_at) FROM groups) AS last_indexed_at,
|
||||
(SELECT json_agg(json_build_object('source', source, 'n', n) ORDER BY n DESC)
|
||||
FROM (SELECT source, count(*) AS n FROM sample GROUP BY 1) s) AS sources`)
|
||||
const r = rows[0]
|
||||
const releases = Math.max(0, Number(r.releases))
|
||||
const sampled = Number(r.sampled)
|
||||
// Sample counts scaled to all visible releases.
|
||||
const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
|
||||
return {
|
||||
approximate: true,
|
||||
releases,
|
||||
readableReleases: scale(r.sampled_readable),
|
||||
files: Math.max(0, Number(r.files)),
|
||||
size: scale(r.sampled_size),
|
||||
encryptedReleases: scale(r.sampled_encrypted),
|
||||
lastIndexedAt: r.last_indexed_at,
|
||||
sources: (r.sources ?? []).map((s: any) => ({ source: s.source, releases: scale(s.n) })),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new SearchService()
|
||||
@@ -0,0 +1,132 @@
|
||||
import nntpConfig from '#config/nntp'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import { XoverClient } from '#lib/nntp/xover_client'
|
||||
|
||||
interface Slot {
|
||||
client: XoverClient
|
||||
group?: string
|
||||
usedAt: number
|
||||
}
|
||||
|
||||
/**
|
||||
* A pooled connection unused this long is replaced: the news server drops
|
||||
* idle connections without telling, and a request on one then hangs until
|
||||
* the inactivity timeout (LOG.md 2026-10-05 (11)).
|
||||
*/
|
||||
const IDLE_MS = 60 * 1000
|
||||
|
||||
/**
|
||||
* NNTP connections in the seed API for relay nodes (CONCEPT.md D38): the
|
||||
* indexer fetches overview chunks and passes the gzip stream on unopened, so
|
||||
* nodes need no news server account and every connection comes from this
|
||||
* machine's IP. SEED_NNTP_CONNECTIONS (0: relay off).
|
||||
*/
|
||||
class SeedFetchPool {
|
||||
private idle: Slot[] = []
|
||||
private open = 0
|
||||
private waiters: (() => void)[] = []
|
||||
|
||||
get size() {
|
||||
return indexerConfig.seed.nntpConnections
|
||||
}
|
||||
|
||||
/**
|
||||
* Pooled connections fetch compressed unless SEED_NNTP_COMPRESS=false. A failure is retried once without compression, on a connection of its
|
||||
* own: either the pooled connection was dead (dropped while idle), or the
|
||||
* server stalled mid-reply in compressed mode. Eweka does that for some
|
||||
* chunks: it stops after most of the stream and never sends the rest
|
||||
* (a.b.ath 49701720000: 19,672 of 20,000 overviews, then silence), while
|
||||
* the same chunk uncompressed comes in 0.3 s (LOG.md 2026-10-05 (11)).
|
||||
*/
|
||||
async fetch(group: string, from: number, to: number): Promise<Buffer> {
|
||||
try {
|
||||
return await this.fetchOnce(group, from, to)
|
||||
} catch {
|
||||
return this.fetchPlain(group, from, to)
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchPlain(group: string, from: number, to: number): Promise<Buffer> {
|
||||
const client = await this.connect(false)
|
||||
try {
|
||||
await client.group(group)
|
||||
return await client.xoverRaw(from, to)
|
||||
} finally {
|
||||
client.close()
|
||||
}
|
||||
}
|
||||
|
||||
private connect(compress: boolean) {
|
||||
return XoverClient.connect({
|
||||
host: nntpConfig.host,
|
||||
port: nntpConfig.port,
|
||||
secure: nntpConfig.secure,
|
||||
servername: nntpConfig.tlsServername,
|
||||
user: nntpConfig.user,
|
||||
password: nntpConfig.password?.release(),
|
||||
timeoutMs: indexerConfig.seed.nntpTimeoutMs,
|
||||
compress,
|
||||
})
|
||||
}
|
||||
|
||||
private async fetchOnce(group: string, from: number, to: number): Promise<Buffer> {
|
||||
const slot = await this.acquire()
|
||||
let ok = false
|
||||
try {
|
||||
if (slot.group !== group) {
|
||||
await slot.client.group(group)
|
||||
slot.group = group
|
||||
}
|
||||
const raw = await slot.client.xoverRaw(from, to)
|
||||
ok = true
|
||||
return raw
|
||||
} finally {
|
||||
this.release(slot, ok)
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes the idle connections (tests, shutdown); busy ones close when released. */
|
||||
closeIdle() {
|
||||
for (const slot of this.idle.splice(0)) {
|
||||
slot.client.close()
|
||||
this.open--
|
||||
}
|
||||
}
|
||||
|
||||
private async acquire(): Promise<Slot> {
|
||||
while (true) {
|
||||
const slot = this.idle.pop()
|
||||
if (slot) {
|
||||
if (Date.now() - slot.usedAt < IDLE_MS) return slot
|
||||
slot.client.close()
|
||||
this.open--
|
||||
continue
|
||||
}
|
||||
if (this.open < this.size) {
|
||||
this.open++
|
||||
try {
|
||||
return { client: await this.connect(indexerConfig.seed.nntpCompress), usedAt: Date.now() }
|
||||
} catch (error) {
|
||||
this.open--
|
||||
this.waiters.shift()?.()
|
||||
throw error
|
||||
}
|
||||
}
|
||||
await new Promise<void>((resolve) => this.waiters.push(resolve))
|
||||
}
|
||||
}
|
||||
|
||||
private release(slot: Slot, ok: boolean) {
|
||||
if (ok) {
|
||||
slot.usedAt = Date.now()
|
||||
this.idle.push(slot)
|
||||
} else {
|
||||
// A failed command can leave the connection mid-reply: start afresh.
|
||||
slot.client.close()
|
||||
this.open--
|
||||
}
|
||||
this.waiters.shift()?.()
|
||||
}
|
||||
}
|
||||
|
||||
export default new SeedFetchPool()
|
||||
@@ -0,0 +1,234 @@
|
||||
import { createHash, randomBytes } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { cut, uncovered } from '#lib/seed/plan'
|
||||
import NntpService from '#services/NntpService'
|
||||
import SeedService from '#services/SeedService'
|
||||
import BackfillService from '#services/BackfillService'
|
||||
import SeedFetchPool from '#services/SeedFetchPool'
|
||||
import { articleAtDate } from '#lib/nntp/date_search'
|
||||
import { normalizeSeedFile, type SeedBatch } from '#lib/seed/wire'
|
||||
|
||||
/** A lease without a batch for this long is given to another node; the range resumes at next_id. */
|
||||
export const LEASE_TTL_SECONDS = 15 * 60
|
||||
/** Leases per range before it counts as failed (a range that keeps breaking nodes). */
|
||||
const MAX_ATTEMPTS = 5
|
||||
/** Articles per XOVER probe when looking for an article by date. */
|
||||
const PROBE = 50
|
||||
|
||||
export interface SeedLease {
|
||||
id: number
|
||||
group: string
|
||||
/** Resume point: first article not reported yet. */
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
export class LeaseLost extends Error {}
|
||||
|
||||
const hash = (token: string) => createHash('sha256').update(token).digest('hex')
|
||||
|
||||
/**
|
||||
* The seed orchestrator (CONCEPT.md D38): seeding nodes lease article
|
||||
* ranges over HTTPS, report what they aggregated in batches (D37), and
|
||||
* finish or give back the range. Nodes only make outgoing requests; a lease
|
||||
* that goes quiet for LEASE_TTL_SECONDS passes to the next node that asks.
|
||||
*/
|
||||
class SeedOrchestrator {
|
||||
/** Creates a node, or gives an existing one a new token. Returns the token (stored hashed only). */
|
||||
async issueToken(name: string): Promise<string> {
|
||||
const token = `sn_${randomBytes(24).toString('base64url')}`
|
||||
await db
|
||||
.table('seed_nodes')
|
||||
.insert({ name, token_hash: hash(token), disabled: false })
|
||||
.onConflict('name')
|
||||
.merge({ token_hash: hash(token), disabled: false })
|
||||
return token
|
||||
}
|
||||
|
||||
async nodeForToken(token: string): Promise<{ id: number; name: string } | null> {
|
||||
const node = await db.from('seed_nodes').where('token_hash', hash(token)).where('disabled', false).first()
|
||||
if (!node) return null
|
||||
if (!node.last_seen_at || Date.now() - new Date(node.last_seen_at).getTime() > 60_000) {
|
||||
await db.from('seed_nodes').where('id', node.id).update({ last_seen_at: new Date() })
|
||||
}
|
||||
return { id: Number(node.id), name: node.name }
|
||||
}
|
||||
|
||||
/**
|
||||
* The next range for this node, oldest plan first; a quiet lease is taken
|
||||
* over. Nothing while the database is over the backfill's disk limits: the
|
||||
* node asks again a minute later.
|
||||
*/
|
||||
async lease(nodeId: number): Promise<SeedLease | null> {
|
||||
if ((await BackfillService.room()) !== true) return null
|
||||
// Fully reported but never finished (the node went away at the very end): nothing left to fetch.
|
||||
await db.rawQuery(
|
||||
`UPDATE seed_ranges SET status = 'done', done_at = now(), node_id = NULL
|
||||
WHERE next_id > end_id AND (status = 'queued' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))`,
|
||||
[LEASE_TTL_SECONDS]
|
||||
)
|
||||
// Ranges that broke node after node stop being handed out.
|
||||
await db.rawQuery(
|
||||
`UPDATE seed_ranges SET status = 'failed', error = coalesce(error, 'lease expired too often')
|
||||
WHERE status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?) AND attempts >= ?`,
|
||||
[LEASE_TTL_SECONDS, MAX_ATTEMPTS]
|
||||
)
|
||||
// Priorities take turns: while ranges of a higher priority are leased for less than half of
|
||||
// the work in progress they go first, otherwise the rest does (a 2026 gap fetches ~5x slower
|
||||
// than older history; LOG.md 2026-10-06 (10)).
|
||||
const { rows: mix } = await db.rawQuery(
|
||||
`SELECT count(*) FILTER (WHERE priority > 0) AS high, count(*) AS all FROM seed_ranges
|
||||
WHERE status = 'leased' AND heartbeat_at >= now() - make_interval(secs => ?)`,
|
||||
[LEASE_TTL_SECONDS]
|
||||
)
|
||||
const order = Number(mix[0].high) * 2 < Number(mix[0].all) + 1 ? 'priority DESC, id' : 'priority ASC, id'
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE seed_ranges SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1
|
||||
WHERE id = (
|
||||
SELECT id FROM seed_ranges
|
||||
WHERE status = 'queued' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?))
|
||||
ORDER BY ${order} LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
RETURNING id, group_name, next_id, end_id`,
|
||||
[nodeId, LEASE_TTL_SECONDS]
|
||||
)
|
||||
const row = rows[0]
|
||||
return row ? { id: Number(row.id), group: row.group_name, start: Number(row.next_id), end: Number(row.end_id) } : null
|
||||
}
|
||||
|
||||
/** Stores a node's batch and moves the range's resume point; LeaseLost when the range is no longer this node's. */
|
||||
async batch(nodeId: number, rangeId: number, batch: SeedBatch) {
|
||||
const range = await this.leased(nodeId, rangeId)
|
||||
const stored = await SeedService.store(batch.files.map(normalizeSeedFile))
|
||||
await SeedService.storeFragments(batch.fragments, { start: Number(range.start_id), end: Number(range.end_id) })
|
||||
await db.rawQuery(
|
||||
`UPDATE seed_ranges SET next_id = greatest(next_id, least(?, end_id + 1)), heartbeat_at = now(),
|
||||
headers = headers + ?, files = files + ?, fragments = fragments + ?
|
||||
WHERE id = ?`,
|
||||
[batch.next, batch.headers, batch.files.length, batch.fragments.length, rangeId]
|
||||
)
|
||||
return stored
|
||||
}
|
||||
|
||||
/**
|
||||
* Relay: overviews of `from`..`to` of a leased range, fetched here and handed
|
||||
* on as the gzip stream the news server sent. Keeps the lease alive.
|
||||
*/
|
||||
async chunk(nodeId: number, rangeId: number, from: number, to: number): Promise<Buffer> {
|
||||
const range = await this.leased(nodeId, rangeId)
|
||||
const start = Math.max(from, Number(range.start_id))
|
||||
const end = Math.min(to, Number(range.end_id), start + 49_999)
|
||||
if (end < start) throw new RangeError(`Articles ${from}–${to} are outside range ${rangeId}`)
|
||||
await db.from('seed_ranges').where('id', rangeId).update({ heartbeat_at: new Date() })
|
||||
return SeedFetchPool.fetch(range.group_name, start, end)
|
||||
}
|
||||
|
||||
async finish(nodeId: number, rangeId: number) {
|
||||
await this.leased(nodeId, rangeId)
|
||||
await db.from('seed_ranges').where('id', rangeId).update({ status: 'done', done_at: new Date(), heartbeat_at: new Date(), error: null })
|
||||
}
|
||||
|
||||
/**
|
||||
* A node gives a range back (it failed, or is shutting down); it's queued
|
||||
* again unless it failed too often. `uncounted`: the node never started on
|
||||
* it (couldn't reach its news server), so the lease doesn't count as an attempt.
|
||||
*/
|
||||
async release(nodeId: number, rangeId: number, error?: string, uncounted = false) {
|
||||
const range = await this.leased(nodeId, rangeId)
|
||||
const attempts = uncounted ? Math.max(0, range.attempts - 1) : range.attempts
|
||||
await db
|
||||
.from('seed_ranges')
|
||||
.where('id', rangeId)
|
||||
.update({ status: attempts >= MAX_ATTEMPTS ? 'failed' : 'queued', node_id: null, attempts, error: error?.slice(0, 2000) ?? null })
|
||||
}
|
||||
|
||||
private async leased(nodeId: number, rangeId: number) {
|
||||
const range = await db.from('seed_ranges').where('id', rangeId).first()
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId) throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
|
||||
return range
|
||||
}
|
||||
|
||||
/**
|
||||
* Plans ranges for groups: from below what live indexing and the backfill
|
||||
* cover (or below earlier seed ranges) down to `days` back, or to the
|
||||
* server's oldest article. A group's backfill stops where its seed plan
|
||||
* begins. Ranges are interleaved across groups, newest first.
|
||||
*/
|
||||
async plan(
|
||||
groups: string[],
|
||||
options: { days?: number; size: number; priority?: number }
|
||||
): Promise<{ group: string; from: number; to: number; ranges: number; gaps: number }[]> {
|
||||
const perGroup: { group: string; ranges: [number, number][] }[] = []
|
||||
const summary: { group: string; from: number; to: number; ranges: number; gaps: number }[] = []
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
for (const name of groups) {
|
||||
const group = await db.from('groups').where('name', name).first()
|
||||
if (!group) throw new Error(`Unknown group ${name}`)
|
||||
const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, name), `GROUP ${name}`)
|
||||
const dateOf = async (n: bigint) => {
|
||||
const { overviews } = await NntpService.timed<any>(conn.xover(n, n + BigInt(PROBE)), `XOVER ${name} ${n}`)
|
||||
const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
|
||||
const date = header ? Date.parse(header.date) : Number.NaN
|
||||
return Number.isNaN(date) ? null : date
|
||||
}
|
||||
// Top: where live indexing began, or as far down as the backfill has come.
|
||||
let top = Number(await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf))
|
||||
if (group.backfill_next_id !== null) top = Math.min(top, Number(group.backfill_next_id))
|
||||
const backfilling = group.backfill_next_id !== null && group.backfill_until_id !== null && Number(group.backfill_next_id) > Number(group.backfill_until_id)
|
||||
const bottom = options.days
|
||||
? Number(await articleAtDate(first, last, Date.now() - options.days * 24 * 60 * 60 * 1000, dateOf))
|
||||
: Number(first)
|
||||
|
||||
// Everything below the top that no earlier range covers (in any state). Planning below
|
||||
// the lowest earlier range left a 13-month gap in a.b.ath (LOG.md 2026-10-06 (7)).
|
||||
const planned = await db.from('seed_ranges').where('group_name', name).select('start_id', 'end_id')
|
||||
const gaps = uncovered(bottom, top - 1, planned.map((r) => [Number(r.start_id), Number(r.end_id)] as [number, number]))
|
||||
const ranges = cut(gaps, options.size)
|
||||
perGroup.push({ group: name, ranges })
|
||||
summary.push({ group: name, from: bottom, to: top - 1, ranges: ranges.length, gaps: gaps.length })
|
||||
// The seed plan takes over the rest of this group's backfill.
|
||||
if (backfilling && ranges.length > 0) {
|
||||
await db.from('groups').where('id', group.id).update({ backfill_until_id: group.backfill_next_id })
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
|
||||
// Round-robin over groups, each newest first, so every group fills in from the present backwards.
|
||||
const rows: Record<string, unknown>[] = []
|
||||
for (let i = 0; perGroup.some((g) => i < g.ranges.length); i++) {
|
||||
for (const g of perGroup) {
|
||||
const range = g.ranges[i]
|
||||
if (range) rows.push({ group_name: g.group, start_id: range[0], end_id: range[1], next_id: range[0], priority: options.priority ?? 0 })
|
||||
}
|
||||
}
|
||||
for (let i = 0; i < rows.length; i += 5000) await db.table('seed_ranges').multiInsert(rows.slice(i, i + 5000))
|
||||
return summary
|
||||
}
|
||||
|
||||
async status() {
|
||||
const { rows: ranges } = await db.rawQuery(
|
||||
`SELECT status, count(*)::int AS ranges, sum(end_id - start_id + 1)::bigint AS articles,
|
||||
sum(next_id - start_id)::bigint AS reached, sum(headers)::bigint AS headers, sum(files)::bigint AS files
|
||||
FROM seed_ranges GROUP BY status ORDER BY status`
|
||||
)
|
||||
const { rows: nodes } = await db.rawQuery(
|
||||
`SELECT n.name, n.last_seen_at, n.disabled,
|
||||
(SELECT count(*)::int FROM seed_ranges r WHERE r.node_id = n.id AND r.status = 'leased') AS leases,
|
||||
(SELECT coalesce(sum(headers), 0)::bigint FROM seed_ranges r WHERE r.node_id = n.id) AS headers
|
||||
FROM seed_nodes n ORDER BY n.name`
|
||||
)
|
||||
const { rows: recent } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(headers), 0)::bigint AS headers FROM seed_ranges WHERE done_at > now() - interval '1 hour'`
|
||||
)
|
||||
const room = await BackfillService.room()
|
||||
return { ranges, nodes, headersDoneLastHour: Number(recent[0].headers), paused: room === true ? null : room }
|
||||
}
|
||||
}
|
||||
|
||||
export default new SeedOrchestrator()
|
||||
@@ -0,0 +1,152 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import IngestService from '#services/IngestService'
|
||||
import FileStore, { insertLimits } from '#services/FileStore'
|
||||
import SfvService from '#services/SfvService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import { fromFragment, type SeedFile, type SeedFragment } from '#lib/seed/wire'
|
||||
import InspectionQueue from '#services/InspectionQueue'
|
||||
import GarbageFilter from '#services/GarbageFilter'
|
||||
|
||||
/** Articles from a range's edge within which a file may continue in the neighbouring range. */
|
||||
const EDGE_ARTICLES = 50_000
|
||||
|
||||
export interface SeedStoreResult {
|
||||
files: number
|
||||
/** Files that existed already (same file_key, e.g. from live indexing or a crosspost). */
|
||||
existing: number
|
||||
releases: number
|
||||
inspections: number
|
||||
/** Random-named files of garbage signatures or dropped releases, not stored (D42). */
|
||||
dropped: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores what seeding nodes found (CONCEPT.md D37): per complete file its
|
||||
* release, the file with its segment list, and one completed binary, as
|
||||
* worker:file would have left it, so par2 naming, inspection and GC treat
|
||||
* it like any other file; fragments take the regular ingest path. A batch
|
||||
* is a few statements, not one per article.
|
||||
*/
|
||||
class SeedService {
|
||||
async store(batch: SeedFile[]): Promise<SeedStoreResult> {
|
||||
const result: SeedStoreResult = { files: 0, existing: 0, releases: 0, inspections: 0, dropped: 0 }
|
||||
// A repost can complete the same file twice in one batch; ON CONFLICT DO UPDATE takes each row once.
|
||||
const unique = [...new Map(batch.map((file) => [file.fileKey, file])).values()]
|
||||
// Garbage isn't stored at all: no release, file or binary (CONCEPT.md D42).
|
||||
const { kept: files, dropped } = await GarbageFilter.keep(unique, (file) => ({
|
||||
name: file.name,
|
||||
messageId: file.segments[0]?.[1],
|
||||
groups: file.groups,
|
||||
releaseKey: file.release?.key ?? file.fileKey,
|
||||
}))
|
||||
result.dropped = dropped
|
||||
if (files.length === 0) return result
|
||||
const knex = db.connection().getWriteClient()
|
||||
const now = new Date()
|
||||
const postedOf = (file: SeedFile) => (file.postedAt ? new Date(file.postedAt) : undefined)
|
||||
|
||||
const releaseIds = await IngestService.upsertReleases(
|
||||
knex,
|
||||
files.map((file) => ({ release: file.release, poster: file.poster, groups: file.groups, postedAt: postedOf(file) })),
|
||||
now
|
||||
)
|
||||
result.releases = releaseIds.size
|
||||
|
||||
const rows = files.map((file) => ({
|
||||
file_key: file.fileKey,
|
||||
filename: file.name,
|
||||
poster: file.poster,
|
||||
date: (postedOf(file) ?? now).getTime(),
|
||||
parts: file.segments.length,
|
||||
segments: file.segments.map(([number, id, size]) => ({ number, id, size })),
|
||||
groups: JSON.stringify(file.groups),
|
||||
release_id: file.release ? (releaseIds.get(file.release.key) ?? null) : null,
|
||||
size: file.segments.reduce((sum, [, , size]) => sum + size, 0),
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}))
|
||||
const inserted = await FileStore.insert(rows)
|
||||
result.files = inserted.length
|
||||
result.existing = rows.length - inserted.length
|
||||
|
||||
// File ids for the binaries below (files queued for inspection only): new ones from the insert,
|
||||
// existing ones looked up. Looking up every key of the batch probed all files partitions again,
|
||||
// 15% of Postgres' active time (LOG.md 2026-10-06 (12)).
|
||||
const fileIds = new Map<string, number>()
|
||||
for (const row of inserted) if (row.file_key) fileIds.set(row.file_key, row.id)
|
||||
const missing = files.filter((file) => !fileIds.has(file.fileKey) && ContentService.isCandidate(file.name)).map((file) => file.fileKey)
|
||||
for (let i = 0; i < missing.length; i += 5000) {
|
||||
for (const row of await db.from('files').whereIn('file_key', missing.slice(i, i + 5000)).select('id', 'file_key')) {
|
||||
fileIds.set(row.file_key, Number(row.id))
|
||||
}
|
||||
}
|
||||
// A completed binary, as worker:file would have left behind, only for files queued for content
|
||||
// inspection: par2 naming (D16) needs the fingerprint inspection stores on it. For the others
|
||||
// (~85% in a.b.ath) it was only GC churn: 1.6M rows an hour inserted and deleted (LOG.md 2026-10-06 (11)).
|
||||
const insertedKeys = new Set(inserted.map((row) => row.file_key))
|
||||
const binaries = files.flatMap((file, i) => {
|
||||
if (!ContentService.isCandidate(file.name)) return []
|
||||
return [{
|
||||
file_key: file.fileKey,
|
||||
poster: file.poster,
|
||||
subject: file.subject,
|
||||
name: file.name,
|
||||
total_parts: file.totalParts,
|
||||
groups: rows[i].groups,
|
||||
posted_at: postedOf(file) ?? null,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
completed_at: now,
|
||||
release_id: rows[i].release_id,
|
||||
file_id: fileIds.get(file.fileKey) ?? null,
|
||||
// New files are queued for inspection below; an existing file was queued when it was added.
|
||||
awaiting_inspection: insertedKeys.has(file.fileKey) ? true : null,
|
||||
}]
|
||||
})
|
||||
binaries.sort((a, b) => a.file_key.localeCompare(b.file_key))
|
||||
for (let i = 0; i < binaries.length; i += insertLimits.rows) {
|
||||
await knex('binaries')
|
||||
.insert(binaries.slice(i, i + insertLimits.rows))
|
||||
.onConflict('file_key')
|
||||
.merge({
|
||||
file_id: knex.raw('coalesce(binaries.file_id, excluded.file_id)') as any,
|
||||
awaiting_inspection: knex.raw('coalesce(binaries.awaiting_inspection, excluded.awaiting_inspection)') as any,
|
||||
})
|
||||
}
|
||||
|
||||
// As worker:file does for a new file: an SFV posted earlier may list it (D34), then inspection.
|
||||
const rowByKey = new Map(rows.map((row) => [row.file_key, row]))
|
||||
const added = inserted.flatMap((file) => {
|
||||
const row = file.file_key ? rowByKey.get(file.file_key) : undefined
|
||||
return row ? [{ id: file.id, name: row.filename, poster: row.poster, date: row.date, release_id: row.release_id }] : []
|
||||
})
|
||||
await SfvService.claimAll(added)
|
||||
const candidates = added
|
||||
.filter((file) => ContentService.isCandidate(file.name))
|
||||
.map((file) => file.id)
|
||||
if (candidates.length > 0) await InspectionQueue.add(candidates)
|
||||
result.inspections = candidates.length
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* Segments of files that didn't complete on a node. Only files near the
|
||||
* range's edges can continue in the neighbouring range (worked on by some
|
||||
* node, maybe hours later): those go straight into binaries and parts
|
||||
* (staging would drop them after 30 minutes). The rest are files with
|
||||
* articles missing for good, and are dropped: staging them filled 22 GB
|
||||
* of churn on the NVMe (LOG.md 2026-10-05 (10)). Returns the segments kept.
|
||||
*/
|
||||
async storeFragments(fragments: SeedFragment[], range: { start: number; end: number }): Promise<number> {
|
||||
const nearEdge = new Set<string>()
|
||||
for (const fragment of fragments) {
|
||||
const n = Number(fragment.articleNumber)
|
||||
if (Number.isFinite(n) && (n < range.start + EDGE_ARTICLES || n > range.end - EDGE_ARTICLES)) nearEdge.add(fragment.fileKey)
|
||||
}
|
||||
const kept = fragments.filter((fragment) => nearEdge.has(fragment.fileKey))
|
||||
if (kept.length > 0) await IngestService.ingest(kept.map(fromFragment))
|
||||
return kept.length
|
||||
}
|
||||
}
|
||||
|
||||
export default new SeedService()
|
||||
@@ -0,0 +1,36 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { sevenZipVolume } from '#lib/sevenzip/index'
|
||||
|
||||
/**
|
||||
* The database side of 7z listings (CONCEPT.md D19): which volumes make up
|
||||
* an archive. Listing itself (first segment, tail, bsdtar) is
|
||||
* lib/sevenzip/list.ts, run by inspection here or on a node.
|
||||
*/
|
||||
class SevenZipService {
|
||||
/**
|
||||
* The volumes of the archive `file` belongs to, in order, when all of them
|
||||
* up to the highest number seen are there; null otherwise. An unsplit (or
|
||||
* unnamed) archive is its own only volume.
|
||||
*/
|
||||
async volumes(file: any): Promise<any[] | null> {
|
||||
const name = String(file.real_name ?? file.filename)
|
||||
const volume = sevenZipVolume(name)
|
||||
if (!volume) return [file]
|
||||
if (!file.release_id) return null
|
||||
const base = name.replace(/\.\d+$/, '').toLowerCase()
|
||||
const rows = await db
|
||||
.from('files')
|
||||
.where('release_id', file.release_id)
|
||||
.whereRaw(`lower(coalesce(real_name, filename)) LIKE ? ESCAPE '\\'`, [`${base.replace(/[\\%_]/g, '\\$&')}.%`])
|
||||
const byNumber = new Map<number, any>()
|
||||
for (const row of rows) {
|
||||
const n = sevenZipVolume(String(row.real_name ?? row.filename))
|
||||
if (n && !byNumber.has(n)) byNumber.set(n, row)
|
||||
}
|
||||
const max = Math.max(0, ...byNumber.keys())
|
||||
const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1))
|
||||
return max > 0 && ordered.every(Boolean) ? ordered : null
|
||||
}
|
||||
}
|
||||
|
||||
export default new SevenZipService()
|
||||
@@ -0,0 +1,96 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import FileStore from '#services/FileStore'
|
||||
import { parseSfv, sfvReleaseName } from '#lib/content/index'
|
||||
|
||||
/** Releases made by name rules alone; an SFV's listing outranks them. */
|
||||
const WEAK_SOURCES = ['stem', 'weak-stem', 'subject']
|
||||
/** Files further apart than this from the SFV in posting time aren't taken. */
|
||||
const WINDOW_MS = 2 * 24 * 60 * 60 * 1000
|
||||
|
||||
/**
|
||||
* SFV grouping (CONCEPT.md D34). An uploader that posts every track of an
|
||||
* album as its own file (13.5k such releases from one poster on homelab)
|
||||
* still posts the album's SFV, which names them all.
|
||||
*/
|
||||
class SfvService {
|
||||
/**
|
||||
* Records an SFV file's listing on its release, moves the listed files that
|
||||
* sit in weak releases into it, and names the release after the album when
|
||||
* it's still named after a track or the SFV itself. Returns how many files
|
||||
* moved, or null when it isn't an SFV.
|
||||
*/
|
||||
async attach(sfvFile: any, text: string): Promise<number | null> {
|
||||
const entries = parseSfv(text)
|
||||
if (entries.length === 0 || !sfvFile.release_id) return null
|
||||
const releaseId = Number(sfvFile.release_id)
|
||||
const postedAt = sfvFile.date ? new Date(Number(sfvFile.date)) : null
|
||||
await db
|
||||
.knexQuery()
|
||||
.table('sfv_entries')
|
||||
.insert(entries.map((e) => ({ release_id: releaseId, name: e.name, crc: e.crc, poster: sfvFile.poster, posted_at: postedAt })))
|
||||
.onConflict(['release_id', 'name'])
|
||||
.ignore()
|
||||
await FileStore.setDetails(Number(sfvFile.id), { content_type: 'sfv' })
|
||||
|
||||
const candidates = await db
|
||||
.from('files as f')
|
||||
.join('releases as r', 'r.id', 'f.release_id')
|
||||
.whereIn(db.raw('coalesce(f.real_name, f.filename)') as any, entries.map((e) => e.name))
|
||||
.whereNot('f.release_id', releaseId)
|
||||
.whereIn('r.source', WEAK_SOURCES)
|
||||
.select('f.id', 'f.poster', 'f.date')
|
||||
const moving = candidates.filter((f) => this.sameUpload(f, sfvFile.poster, postedAt))
|
||||
await this.move(moving.map((f) => Number(f.id)), releaseId)
|
||||
|
||||
const release = await db.from('releases').where('id', releaseId).first()
|
||||
const album = sfvReleaseName(String(sfvFile.real_name ?? sfvFile.filename))
|
||||
if (release && WEAK_SOURCES.includes(release.source) && album !== release.name && /^\d{1,3}-/.test(release.name)) {
|
||||
await db.from('releases').where('id', releaseId).update({ name: album })
|
||||
}
|
||||
logger.info(`SFV "${sfvFile.filename}": ${entries.length} files listed, ${moving.length} moved into release ${releaseId} "${album}".`)
|
||||
return moving.length
|
||||
}
|
||||
|
||||
/** A new file an SFV already listed joins that SFV's release. Returns the release, or null. */
|
||||
/**
|
||||
* claim() for a batch of new files: one lookup for all their names, the
|
||||
* per-file work only for the few an SFV lists (seeding stores thousands
|
||||
* of files per batch; CONCEPT.md D37).
|
||||
*/
|
||||
async claimAll(files: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }[]): Promise<number> {
|
||||
if (files.length === 0) return 0
|
||||
const listed = new Set(
|
||||
(await db.from('sfv_entries').whereIn('name', [...new Set(files.map((f) => f.name))]).select('name')).map((e) => e.name as string)
|
||||
)
|
||||
let moved = 0
|
||||
for (const file of files) if (listed.has(file.name) && (await this.claim(file)) !== null) moved++
|
||||
return moved
|
||||
}
|
||||
|
||||
async claim(file: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }): Promise<number | null> {
|
||||
const entries = await db.from('sfv_entries').where('name', file.name)
|
||||
const entry = entries.find((e) => this.sameUpload(file, e.poster, e.posted_at ? new Date(e.posted_at) : null))
|
||||
if (!entry || Number(entry.release_id) === Number(file.release_id)) return null
|
||||
if (file.release_id) {
|
||||
const current = await db.from('releases').where('id', file.release_id).first()
|
||||
if (current && !WEAK_SOURCES.includes(current.source)) return null
|
||||
}
|
||||
await this.move([file.id], Number(entry.release_id))
|
||||
return Number(entry.release_id)
|
||||
}
|
||||
|
||||
private sameUpload(file: { poster?: string | null; date?: number | string | null }, poster: string | null, postedAt: Date | null) {
|
||||
if (poster && file.poster && file.poster !== poster) return false
|
||||
if (postedAt && file.date && Math.abs(Number(file.date) - postedAt.getTime()) > WINDOW_MS) return false
|
||||
return true
|
||||
}
|
||||
|
||||
private async move(fileIds: number[], releaseId: number) {
|
||||
if (fileIds.length === 0) return
|
||||
await db.from('files').whereIn('id', fileIds).update({ release_id: releaseId })
|
||||
await db.from('binaries').whereIn('file_id', fileIds).update({ release_id: releaseId })
|
||||
}
|
||||
}
|
||||
|
||||
export default new SfvService()
|
||||
@@ -0,0 +1,45 @@
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
|
||||
/** After a signal, give cleanup this long before exiting anyway. */
|
||||
const FORCE_EXIT_MS = 25_000
|
||||
|
||||
/**
|
||||
* Graceful shutdown for the long-running commands. SIGTERM (docker stop) and
|
||||
* SIGINT wake up `wait()` and `sleep()`, so workers can finish their active
|
||||
* jobs and close their connections before the process exits.
|
||||
*/
|
||||
class Shutdown {
|
||||
private readonly controller = new AbortController()
|
||||
|
||||
constructor() {
|
||||
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
||||
process.once(signal, () => {
|
||||
this.controller.abort()
|
||||
setTimeout(() => process.exit(0), FORCE_EXIT_MS).unref()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
get requested(): boolean {
|
||||
return this.controller.signal.aborted
|
||||
}
|
||||
|
||||
/** Resolves when a shutdown signal arrives. */
|
||||
wait(): Promise<void> {
|
||||
if (this.requested) return Promise.resolve()
|
||||
return new Promise((resolve) =>
|
||||
this.controller.signal.addEventListener('abort', () => resolve(), { once: true })
|
||||
)
|
||||
}
|
||||
|
||||
/** Sleeps, but returns early on shutdown. */
|
||||
async sleep(ms: number): Promise<void> {
|
||||
try {
|
||||
await sleep(ms, undefined, { signal: this.controller.signal })
|
||||
} catch {
|
||||
// aborted by a shutdown signal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new Shutdown()
|
||||
@@ -0,0 +1,146 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import IngestService from '#services/IngestService'
|
||||
import NameService from '#services/NameService'
|
||||
import { splitVerified } from '#lib/split/index'
|
||||
|
||||
/** Conflicted binaries handled per run; the rest follow on the next tick. */
|
||||
const BATCH = 200
|
||||
|
||||
export interface SplitResult {
|
||||
parents: number
|
||||
volumes: number
|
||||
completed: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits conflicted obfuscated binaries (several equal-sized files under one
|
||||
* size+parts identity) into one binary per volume. Only volumes whose
|
||||
* combined part CRCs match the file CRC of their last part are created, so a
|
||||
* split can never produce a corrupt file; the rest waits for more articles.
|
||||
* See CONCEPT.md (D20).
|
||||
*/
|
||||
class SplitService {
|
||||
async splitPending(): Promise<SplitResult> {
|
||||
const parents = await db
|
||||
.from('binaries')
|
||||
.whereNotNull('conflicted_at')
|
||||
.whereNull('parent_binary_id')
|
||||
// split_at = 'infinity' marks binaries that cannot be split (see migration …810).
|
||||
.where((q) => q.whereNull('split_at').orWhereRaw('last_seen_at > split_at'))
|
||||
// Most recently active first: those are the posts still coming in.
|
||||
.orderBy('last_seen_at', 'desc')
|
||||
.limit(BATCH)
|
||||
|
||||
const volumeIds: number[] = []
|
||||
for (const parent of parents) {
|
||||
volumeIds.push(...(await this.split(parent)))
|
||||
}
|
||||
|
||||
// Former conflicts that turned out to be reposts (unflagged, keep their
|
||||
// split_at): no new article may arrive to trigger their completion.
|
||||
const unflagged = await db
|
||||
.from('binaries')
|
||||
.whereNull('conflicted_at')
|
||||
.whereNull('completed_at')
|
||||
.whereNull('parent_binary_id')
|
||||
.whereNotNull('split_at')
|
||||
.whereRaw("split_at <> 'infinity'")
|
||||
.limit(BATCH)
|
||||
.select('id')
|
||||
volumeIds.push(...unflagged.map((b) => Number(b.id)))
|
||||
|
||||
await IngestService.matchPar2Releases(volumeIds)
|
||||
await NameService.resolve(volumeIds)
|
||||
const completed = await IngestService.claimCompleted(volumeIds)
|
||||
await IngestService.enqueue(completed)
|
||||
return { parents: parents.length, volumes: volumeIds.length, completed: completed.length }
|
||||
}
|
||||
|
||||
/** Creates the newly verified volumes of one conflicted binary; returns their ids. */
|
||||
private async split(parent: any): Promise<number[]> {
|
||||
return db.transaction(async (trx) => {
|
||||
// Unverified volumes from before verification existed can never complete.
|
||||
await trx.from('binaries').where('parent_binary_id', parent.id).whereNull('completed_at').delete()
|
||||
|
||||
const { rows } = await trx.rawQuery(
|
||||
`WITH taken AS (
|
||||
SELECT p.message_id FROM parts p JOIN binaries v ON v.id = p.binary_id
|
||||
WHERE v.parent_binary_id = ?
|
||||
), candidates AS (
|
||||
SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM parts WHERE binary_id = ?
|
||||
UNION ALL
|
||||
SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM part_conflicts WHERE binary_id = ?
|
||||
)
|
||||
SELECT * FROM candidates WHERE message_id NOT IN (SELECT message_id FROM taken)`,
|
||||
[parent.id, parent.id, parent.id]
|
||||
)
|
||||
const volumes = splitVerified(
|
||||
rows.map((r: any) => ({
|
||||
number: r.number,
|
||||
messageId: r.message_id,
|
||||
bytes: r.bytes,
|
||||
articleNumber: r.article_number === null ? null : Number(r.article_number),
|
||||
pcrc32: r.pcrc32,
|
||||
partSize: r.part_size,
|
||||
fileCrc32: r.file_crc32,
|
||||
})),
|
||||
{ totalParts: parent.total_parts }
|
||||
)
|
||||
|
||||
const now = new Date()
|
||||
const ids: number[] = []
|
||||
for (const articles of volumes) {
|
||||
// A verified volume is fixed, so its first article identifies it.
|
||||
const fileKey = createHash('sha1').update(`${parent.file_key}#${articles[0].messageId}`).digest('hex')
|
||||
const inserted = await trx
|
||||
.knexClient('binaries')
|
||||
.insert({
|
||||
file_key: fileKey,
|
||||
poster: parent.poster,
|
||||
subject: parent.subject,
|
||||
name: parent.name,
|
||||
total_parts: parent.total_parts,
|
||||
groups: JSON.stringify(parent.groups),
|
||||
posted_at: parent.posted_at,
|
||||
size: parent.size,
|
||||
release_id: parent.release_id,
|
||||
parent_binary_id: parent.id,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
})
|
||||
.onConflict('file_key')
|
||||
.ignore()
|
||||
.returning('id')
|
||||
if (inserted.length === 0) continue
|
||||
const id = Number(inserted[0].id)
|
||||
await trx.knexClient('parts').insert(
|
||||
articles.map((a) => ({
|
||||
binary_id: id,
|
||||
number: a.number,
|
||||
message_id: a.messageId,
|
||||
bytes: a.bytes,
|
||||
article_number: a.articleNumber,
|
||||
pcrc32: a.pcrc32,
|
||||
part_size: a.partSize,
|
||||
file_crc32: a.fileCrc32,
|
||||
}))
|
||||
)
|
||||
// The fingerprint of a volume is that of its part 1 article.
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries SET md5_16k = (
|
||||
SELECT md5_16k FROM part_conflicts WHERE binary_id = ? AND message_id = ?
|
||||
UNION ALL SELECT md5_16k FROM parts WHERE binary_id = ? AND message_id = ? LIMIT 1)
|
||||
WHERE id = ?`,
|
||||
[parent.id, articles[0].messageId, parent.id, articles[0].messageId, id]
|
||||
)
|
||||
ids.push(id)
|
||||
}
|
||||
|
||||
await trx.from('binaries').where('id', parent.id).update({ split_at: now })
|
||||
return ids
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default new SplitService()
|
||||
@@ -0,0 +1,493 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import NzbImportService from '#services/NzbImportService'
|
||||
import IngestService, { type OverviewHeader } from '#services/IngestService'
|
||||
import { parseNzb } from '#lib/nzb/index'
|
||||
import {
|
||||
moderationTarget,
|
||||
passwordFromSpot,
|
||||
spotterId,
|
||||
decodeSpotBinary,
|
||||
decodeSpotText,
|
||||
parseSpotArticleHeaders,
|
||||
parseSpotHeader,
|
||||
parseSpotXml,
|
||||
verifySpotHeader,
|
||||
type SpotHeader,
|
||||
type SpotXml,
|
||||
} from '#lib/spotnet/index'
|
||||
|
||||
/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
|
||||
const MAX_NZB_SEGMENTS = 20
|
||||
/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
|
||||
const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
|
||||
/** Rows per INSERT: 14 columns each, under Postgres' 65,535 bind parameters. */
|
||||
const INSERT_CHUNK = 2000
|
||||
/** Spot images larger than this many articles aren't fetched. */
|
||||
const MAX_IMAGE_SEGMENTS = 10
|
||||
/** A claimed spot whose worker died is taken again after this long. */
|
||||
const RECLAIM_AFTER = '1 hour'
|
||||
|
||||
/** The NZB's articles were read, but they don't decode to an NZB (e.g. "invalid code lengths set"). */
|
||||
export class NzbUnreadable extends Error {}
|
||||
|
||||
/**
|
||||
* A spot's title: from its XML, which is decoded properly (the Subject of a
|
||||
* raw header is read as ISO-8859-1), else from the header.
|
||||
*/
|
||||
function spotTitle(head: { header: SpotHeader; xml: SpotXml | null }): string {
|
||||
return head.xml?.title || head.header.title
|
||||
}
|
||||
|
||||
/**
|
||||
* A placeholder NZB (e.g. Dummy.rar + Dummy.nzb, 2.6 KB, shared by many
|
||||
* spots): less than 1% of the size the spot announces (the real NZB is
|
||||
* handed out elsewhere), or under 16 KB in all (text-only spots: requests,
|
||||
* announcements). Its files are not imported, and the spot is hidden.
|
||||
*/
|
||||
export function isDummyNzb(nzb: { files: { segments: { bytes: number }[] }[] }, spotSize: number): boolean {
|
||||
const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
|
||||
return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
|
||||
}
|
||||
|
||||
export function isSpotGroup(name: string): boolean {
|
||||
return indexerConfig.spotGroups.includes(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Spotnet (CONCEPT.md D29). Spots are signed posts in free.pt; their full
|
||||
* header holds XML with the message ids of a compressed NZB, which is
|
||||
* imported like a posted NZB (D26). Spot NZBs name many obfuscated posts
|
||||
* (nyuu, ngPost, the skipped multi-group uploader).
|
||||
*/
|
||||
class SpotService {
|
||||
/** Queues the spots among a batch of one spot group's headers. Returns how many. */
|
||||
async recordHeaders(groupName: string, overviews: [number, OverviewHeader][]): Promise<number> {
|
||||
const now = new Date()
|
||||
const rows = []
|
||||
for (const [articleNumber, header] of overviews) {
|
||||
if (!header?.from || !header['message-id']) continue
|
||||
const spot = parseSpotHeader(header.from, header.subject ?? '', header['message-id'], header.date)
|
||||
if (!spot?.title) continue
|
||||
rows.push({
|
||||
message_id: spot.messageId,
|
||||
group_name: groupName,
|
||||
article_number: Number(articleNumber),
|
||||
title: spot.title,
|
||||
tag: spot.tag || null,
|
||||
poster: spot.poster,
|
||||
category: spot.category,
|
||||
subcats: JSON.stringify(spot.subcats),
|
||||
size: spot.size || null,
|
||||
posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
|
||||
key_id: spot.keyId,
|
||||
spotter_id: spotterId(spot.selfSignedPubKey) || null,
|
||||
// From the overview; titles with accents may fail here (UTF-8 decoding)
|
||||
// and are verified again on their raw header by worker:spots.
|
||||
verified: verifySpotHeader(spot),
|
||||
created_at: now,
|
||||
})
|
||||
}
|
||||
for (let i = 0; i < rows.length; i += INSERT_CHUNK) {
|
||||
await db.knexQuery().table('spots').insert(rows.slice(i, i + INSERT_CHUNK)).onConflict('message_id').ignore()
|
||||
}
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes up to `limit` pending spots, marked `processing`: recent ones
|
||||
* (to be imported) first, then the history in posting order, so live spots
|
||||
* don't wait behind a backfill and moderation follows its targets.
|
||||
*/
|
||||
async claim(limit: number): Promise<any[]> {
|
||||
const claim = async (filter: string, bindings: unknown[], n: number) =>
|
||||
(
|
||||
await db.rawQuery(
|
||||
`UPDATE spots SET status = 'processing', processed_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots WHERE ${filter}
|
||||
ORDER BY article_number LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[...bindings, n]
|
||||
)
|
||||
).rows as any[]
|
||||
const rows = await claim(`status = 'pending' AND posted_at >= ?`, [this.importCutoff()], limit)
|
||||
if (rows.length < limit) rows.push(...(await claim(`status = 'pending'`, [], limit - rows.length)))
|
||||
if (rows.length < limit) {
|
||||
rows.push(...(await claim(`status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}'`, [], limit - rows.length)))
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
/** Spots posted before this are listed, not imported: their NZB is read on first download. */
|
||||
importCutoff(): Date {
|
||||
return new Date(Date.now() - indexerConfig.spotImportDays * 24 * 60 * 60 * 1000)
|
||||
}
|
||||
|
||||
/** Old, verified spots that aren't moderation: listed in bulk, without reading anything more. */
|
||||
isListable(spot: any): boolean {
|
||||
return (
|
||||
spot.verified === true &&
|
||||
spot.posted_at !== null &&
|
||||
new Date(spot.posted_at) < this.importCutoff() &&
|
||||
!moderationTarget({ keyId: spot.key_id, title: spot.title })
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the listing releases (source `spot`, no files) for these spots.
|
||||
* The release key is the one an import of the spot's NZB uses, so the
|
||||
* import on first download fills this release.
|
||||
*/
|
||||
async list(spots: any[]): Promise<number> {
|
||||
if (spots.length === 0) return 0
|
||||
const keyOf = (spot: any) => createHash('sha1').update(`nzb\n${spot.message_id}`).digest('hex')
|
||||
const ids = await IngestService.upsertReleases(
|
||||
db.connection().getWriteClient(),
|
||||
spots.map((spot) => ({
|
||||
release: { key: keyOf(spot), name: spot.title, source: 'spot' as const },
|
||||
poster: spot.poster,
|
||||
groups: [spot.group_name],
|
||||
postedAt: spot.posted_at ? new Date(spot.posted_at) : undefined,
|
||||
}))
|
||||
)
|
||||
const rows = spots.map((spot) => ({ message_id: spot.message_id, release_id: ids.get(keyOf(spot)), size: spot.size }))
|
||||
await db.rawQuery(
|
||||
`UPDATE releases r SET size = v.size FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, size bigint)
|
||||
WHERE r.id = v.release_id AND r.file_count = 0 AND v.size IS NOT NULL`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE spots s SET status = 'listed', release_id = v.release_id, processed_at = now()
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, release_id bigint)
|
||||
WHERE s.message_id = v.message_id`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies the spot on its raw header (the signature covers ISO-8859-1
|
||||
* bytes, which the overview, read as UTF-8, may have mangled), then
|
||||
* applies moderation, lists an old spot or imports a recent one's NZB.
|
||||
* Returns the new status.
|
||||
*/
|
||||
async process(spot: any): Promise<string> {
|
||||
const update: Record<string, unknown> = { processed_at: new Date(), error: null }
|
||||
try {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) {
|
||||
update.status = 'unverified'
|
||||
} else if (moderationTarget(head.header)) {
|
||||
update.status = 'moderation'
|
||||
await this.moderate(head.header, moderationTarget(head.header)!)
|
||||
} else {
|
||||
update.title = spotTitle(head)
|
||||
update.poster = head.header.poster
|
||||
update.spotter_id = spotterId(head.header.selfSignedPubKey) || null
|
||||
Object.assign(update, this.details(head.xml))
|
||||
if (new Date(head.header.stamp * 1000) < this.importCutoff()) {
|
||||
await this.list([{ ...spot, title: spotTitle(head), poster: head.header.poster }])
|
||||
update.status = 'listed'
|
||||
} else {
|
||||
Object.assign(update, await this.importNzb(spot.message_id, head))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
update.status = 'failed'
|
||||
update.error = String(error.message).slice(0, 500)
|
||||
logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
if (update.status === 'imported' || update.status === 'listed') await this.applyPassword(spot.message_id)
|
||||
return update.status as string
|
||||
}
|
||||
|
||||
/**
|
||||
* The NZB of a listing, on first download: reads and imports it. Returns
|
||||
* the release that now holds its files (the listing's own, unless the NZB
|
||||
* joined files already known elsewhere), or null.
|
||||
*/
|
||||
async importListing(releaseId: number): Promise<number | null> {
|
||||
const spot = await db.from('spots').where('release_id', releaseId).where('status', 'listed').first()
|
||||
if (!spot) return null
|
||||
return this.reimport(spot)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a spot's NZB again and imports it, so its files end up in the
|
||||
* spot's own release under the spot's title. Returns that release, or null.
|
||||
*/
|
||||
async reimport(spot: any): Promise<number | null> {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) return null
|
||||
const result = await this.importNzb(spot.message_id, head)
|
||||
await db
|
||||
.from('spots')
|
||||
.where('message_id', spot.message_id)
|
||||
.update({ ...result, ...this.details(head.xml), processed_at: new Date() })
|
||||
await this.applyPassword(spot.message_id)
|
||||
return (result.release_id as number | undefined) ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* A password the spotter wrote in the spot's title or description (not in
|
||||
* the NZB) goes on its release, unless the release has one already.
|
||||
*/
|
||||
async applyPassword(messageId: string): Promise<string | null> {
|
||||
const spot = await db.from('spots').where('message_id', messageId).whereNotNull('release_id').first()
|
||||
const password = spot ? passwordFromSpot(spot.title, spot.description) : null
|
||||
if (!password) return null
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', spot.release_id)
|
||||
.whereNull('password')
|
||||
.update({ password, password_source: 'spot' })
|
||||
return password
|
||||
}
|
||||
|
||||
/** Columns for what the spot's XML adds to its overview: description, image, website. */
|
||||
private details(xml: SpotXml | null): Record<string, unknown> {
|
||||
return {
|
||||
description: xml?.description || null,
|
||||
image: xml?.image ? JSON.stringify(xml.image) : null,
|
||||
...(xml?.website ? { website: xml.website } : {}),
|
||||
details_at: new Date(),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A spot row with its description and image, reading its full header
|
||||
* first when that hasn't happened yet (old spots are listed from their
|
||||
* overview only). Returns the row as it is when Usenet can't be reached.
|
||||
*/
|
||||
async withDetails(spot: any): Promise<any> {
|
||||
if (spot.details_at) return spot
|
||||
try {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
const update = head ? this.details(head.xml) : { details_at: new Date() }
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
await this.applyPassword(spot.message_id)
|
||||
return { ...spot, ...update, image: head?.xml?.image ?? null }
|
||||
} catch (error: any) {
|
||||
logger.warn(`Spot ${spot.message_id} details: ${error.message}`)
|
||||
// The article is gone (430): nothing to read later either.
|
||||
if (/^4\d\d/.test(String(error.message))) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
}
|
||||
return spot
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the details of up to `limit` imported spots that don't have them
|
||||
* (imported before details were kept), newest first. Returns how many.
|
||||
* Listed spots are left to their first view: there are millions.
|
||||
* Returns how many were read (0 when Usenet can't be reached).
|
||||
*/
|
||||
async fillDetails(limit: number): Promise<number> {
|
||||
const spots = await db
|
||||
.from('spots')
|
||||
.where('status', 'imported')
|
||||
.whereNull('details_at')
|
||||
.orderBy('posted_at', 'desc')
|
||||
.limit(limit)
|
||||
let read = 0
|
||||
for (const spot of spots) if ((await this.withDetails(spot)).details_at) read++
|
||||
return read
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes up to `limit` listed spots whose article and NZB haven't been read,
|
||||
* newest first, marking them so another reader skips them.
|
||||
*/
|
||||
async claimListings(limit: number): Promise<any[]> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE spots SET nzb_read_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots WHERE status = 'listed' AND nzb_read_at IS NULL
|
||||
ORDER BY posted_at DESC LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[limit]
|
||||
)
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a listed spot's article and NZB without importing its files
|
||||
* (LOG.md 2026-10-08 (10)): the details (description, image), the NZB's
|
||||
* own password (`password_source = 'nzb'`, over one from the spot's text),
|
||||
* else one in the spot's text; a placeholder NZB hides the spot (`dummy`).
|
||||
* Returns what it found. A connection failure gives the spot back for later.
|
||||
*/
|
||||
async readListing(spot: any): Promise<string> {
|
||||
try {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
return 'unverified'
|
||||
}
|
||||
const update: Record<string, unknown> = this.details(head.xml)
|
||||
let outcome = 'nothing'
|
||||
let nzbXml: string | null = null
|
||||
try {
|
||||
nzbXml = await this.readNzb(head)
|
||||
} catch (error: any) {
|
||||
if (error instanceof NzbUnreadable) {
|
||||
// Corrupt: reading it again won't help (LOG.md 2026-10-08 (12)).
|
||||
update.error = `NZB unreadable: ${error.message}`.slice(0, 500)
|
||||
outcome = 'nzb-unreadable'
|
||||
} else if (/^4\d\d/.test(String(error.message))) {
|
||||
// The NZB's articles are gone: the listing can't be downloaded any more.
|
||||
update.error = 'NZB articles gone'
|
||||
outcome = 'nzb-gone'
|
||||
} else {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
const nzb = nzbXml ? parseNzb(nzbXml) : null
|
||||
if (nzb && nzb.files.length > 0 && isDummyNzb(nzb, head.header.size)) {
|
||||
update.status = 'dummy'
|
||||
outcome = 'dummy'
|
||||
} else if (nzb?.meta.password && spot.release_id) {
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', spot.release_id)
|
||||
.where((q) => q.whereNull('password').orWhere('password_source', 'spot'))
|
||||
.update({ password: nzb.meta.password, password_source: 'nzb' })
|
||||
outcome = 'nzb-password'
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
if (outcome !== 'dummy' && (await this.applyPassword(spot.message_id)) && outcome !== 'nzb-password') outcome = 'spot-password'
|
||||
return outcome
|
||||
} catch (error: any) {
|
||||
// The spot's own article is gone (430): nothing to read later either.
|
||||
if (/^4\d\d/.test(String(error.message))) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
return 'gone'
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ nzb_read_at: null })
|
||||
logger.warn(`Spot ${spot.message_id} listing: ${error.message}`)
|
||||
return 'retry'
|
||||
}
|
||||
}
|
||||
|
||||
/** Reads up to `limit` unread listings, `concurrency` at a time. Returns the outcomes counted. */
|
||||
async readListings(limit: number, concurrency: number): Promise<Record<string, number>> {
|
||||
const spots = await this.claimListings(limit)
|
||||
const counts: Record<string, number> = {}
|
||||
let next = 0
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.max(1, concurrency) }, async () => {
|
||||
while (next < spots.length) {
|
||||
const outcome = await this.readListing(spots[next++])
|
||||
counts[outcome] = (counts[outcome] ?? 0) + 1
|
||||
}
|
||||
})
|
||||
)
|
||||
return counts
|
||||
}
|
||||
|
||||
/** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
|
||||
async image(spot: any): Promise<Buffer | null> {
|
||||
const segments: string[] = spot.image?.segments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_IMAGE_SEGMENTS) return null
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
return decodeSpotBinary(bodies, false)
|
||||
}
|
||||
|
||||
/** The spot's raw header, verified; null when it doesn't verify. */
|
||||
private async readHead(messageId: string): Promise<{ header: SpotHeader; xml: SpotXml | null } | null> {
|
||||
const raw = await this.fetch(`HEAD <${messageId}>`)
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
|
||||
const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
|
||||
if (!header || !verifySpotHeader(header)) return null
|
||||
return { header, xml: parseSpotXml(decodeSpotText(xml)) }
|
||||
}
|
||||
|
||||
/** Downloads and decodes the spot's compressed NZB; null when it has none (or a too large one). */
|
||||
private async readNzb(head: { xml: SpotXml | null }): Promise<string | null> {
|
||||
const segments = head.xml?.nzbSegments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return null
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
try {
|
||||
return decodeSpotBinary(bodies).toString('utf8')
|
||||
} catch (error: any) {
|
||||
throw new NzbUnreadable(String(error.message))
|
||||
}
|
||||
}
|
||||
|
||||
/** Downloads the spot's compressed NZB and imports it as a release (D26). */
|
||||
private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
|
||||
const nzbXml = await this.readNzb(head)
|
||||
if (!nzbXml) return { status: 'no-nzb' }
|
||||
const nzb = parseNzb(nzbXml)
|
||||
if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
|
||||
const { header } = head
|
||||
if (isDummyNzb(nzb, header.size)) return { status: 'dummy' }
|
||||
const title = spotTitle(head)
|
||||
const result = await NzbImportService.import(
|
||||
{
|
||||
id: null,
|
||||
filename: `${title}.nzb`,
|
||||
poster: header.poster,
|
||||
groups: [...new Set(nzb.files.flatMap((f) => f.groups))],
|
||||
date: header.stamp * 1000,
|
||||
first_message_id: messageId,
|
||||
},
|
||||
nzbXml,
|
||||
{ name: title, source: 'spot', moveContainer: false, trusted: true }
|
||||
)
|
||||
return result ? { status: 'imported', release_id: result.releaseId } : { status: 'no-nzb' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a delete/dispose/remove: the target spot is marked `removed`
|
||||
* (its release is then hidden from search). As in Spotweb, a personal
|
||||
* dispose must come from the same spotter within 5 days; a moderator's
|
||||
* (signed with Spotnet's key, no personal key) always counts.
|
||||
*/
|
||||
private async moderate(header: SpotHeader, targetId: string) {
|
||||
const target = await db.from('spots').where('message_id', targetId).first()
|
||||
if (!target) return
|
||||
const personal = header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : ''
|
||||
if (personal) {
|
||||
const age = header.stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
|
||||
if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
|
||||
}
|
||||
await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
|
||||
logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
|
||||
// Spots that joined its release (a correction, a repost) take the files over now.
|
||||
if (target.release_id) {
|
||||
const sharing = await db
|
||||
.from('spots')
|
||||
.where('release_id', target.release_id)
|
||||
.where('status', 'imported')
|
||||
.whereNot('message_id', targetId)
|
||||
for (const spot of sharing) await this.reimport(spot)
|
||||
}
|
||||
}
|
||||
|
||||
/** One HEAD or BODY, as raw bytes. */
|
||||
private async fetch(command: string): Promise<Buffer> {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
const { data } = await NntpService.timed<any>(conn._artcmd(command), command)
|
||||
return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
|
||||
} catch (error: any) {
|
||||
// 430 (no such article) is an answer, not a broken connection.
|
||||
failed = !/^4\d\d/.test(String(error.message))
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new SpotService()
|
||||
@@ -0,0 +1,252 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import IngestService, { type IngestSegment } from '#services/IngestService'
|
||||
import { counterFileKey } from '#lib/subject/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
|
||||
/** Rows per statement (a key and a JSON array each). */
|
||||
const CHUNK = 5000
|
||||
|
||||
/**
|
||||
* Staging of lone header segments (CONCEPT.md D35). Uploads that give every
|
||||
* article its own random subject turned each article into a binary, a part
|
||||
* and a release, which GC deleted two hours later: ~3.9M binaries an hour
|
||||
* against ~123k real files. A segment whose file is new and alone in its
|
||||
* fetch batch now waits in `staged_segments` (unlogged, one index) until a
|
||||
* second segment of that file arrives; only then do both go into the real
|
||||
* tables. GC drops what waited longer than STAGING_TTL_SECONDS (30 min).
|
||||
*
|
||||
* Counter grouping (D36): those random-subject uploads still keep poster,
|
||||
* file counter `[i/M]` and part count per file. Staged segments also carry
|
||||
* that key (`counter_key`); once two staged segments from different subjects
|
||||
* share it, they become one file under that key, and later segments with it
|
||||
* join that file directly.
|
||||
*/
|
||||
class StagingService {
|
||||
enabled(groupName: string): boolean {
|
||||
const groups = indexerConfig.stagingGroups
|
||||
return groups.includes('*') || groups.includes(groupName)
|
||||
}
|
||||
|
||||
/** Whether counter grouping (D36) applies to this group. */
|
||||
countersEnabled(groupName: string): boolean {
|
||||
const groups = indexerConfig.counterGroupingGroups
|
||||
return groups.includes('*') || groups.includes(groupName)
|
||||
}
|
||||
|
||||
/**
|
||||
* The segments of a header batch that should be ingested now: the batch's
|
||||
* own, minus lone segments of new files (staged instead), plus staged
|
||||
* segments whose file this batch shows to be real. `staged` counts the
|
||||
* segments that went to staging.
|
||||
*/
|
||||
async filter(segments: IngestSegment[], { counters = false } = {}): Promise<{ ingest: IngestSegment[]; staged: number }> {
|
||||
const byKey = new Map<string, IngestSegment[]>()
|
||||
for (const segment of segments) {
|
||||
const list = byKey.get(segment.fileKey)
|
||||
if (list) list.push(segment)
|
||||
else byKey.set(segment.fileKey, [segment])
|
||||
}
|
||||
|
||||
const ingest: IngestSegment[] = []
|
||||
const lone: IngestSegment[] = []
|
||||
for (const list of byKey.values()) {
|
||||
// Single-segment files are complete; two different articles of a file in one batch make it real.
|
||||
// (One article twice is a crosspost, LOG.md 2026-10-02.)
|
||||
const articles = new Set(list.map((s) => s.messageId)).size
|
||||
if (list[0].totalParts <= 1 || articles >= 2 || list[0].weakIdentity || list[0].size != null) ingest.push(...list)
|
||||
else lone.push(list[0])
|
||||
}
|
||||
|
||||
const counterOf = new Map<IngestSegment, string>()
|
||||
if (counters) {
|
||||
for (const segment of lone) {
|
||||
const key = counterFileKey(segment)
|
||||
if (key) counterOf.set(segment, key)
|
||||
}
|
||||
}
|
||||
|
||||
// Files we already have a binary for (by subject, or by counters) take their segments directly.
|
||||
const known = new Set<string>()
|
||||
const lookup = [...new Set([...lone.map((s) => s.fileKey), ...counterOf.values()])]
|
||||
for (let i = 0; i < lookup.length; i += CHUNK) {
|
||||
const rows = await db.from('binaries').whereIn('file_key', lookup.slice(i, i + CHUNK)).select('file_key')
|
||||
for (const row of rows) known.add(row.file_key)
|
||||
}
|
||||
const toStage: IngestSegment[] = []
|
||||
const knownCounters = new Set<string>()
|
||||
for (const segment of lone) {
|
||||
const counter = counterOf.get(segment)
|
||||
if (known.has(segment.fileKey)) {
|
||||
ingest.push(segment)
|
||||
} else if (counter && known.has(counter)) {
|
||||
ingest.push(asCounterFile(segment, counter))
|
||||
knownCounters.add(counter)
|
||||
} else {
|
||||
toStage.push(segment)
|
||||
}
|
||||
}
|
||||
|
||||
// Stage the rest; a file that now has two staged segments comes out again.
|
||||
const promoted: string[] = []
|
||||
// Key order, so concurrent jobs lock rows in the same order.
|
||||
toStage.sort((a, b) => a.fileKey.localeCompare(b.fileKey))
|
||||
for (let i = 0; i < toStage.length; i += CHUNK) {
|
||||
const rows = toStage.slice(i, i + CHUNK).map((s) => ({
|
||||
file_key: s.fileKey,
|
||||
counter_key: counterOf.get(s) ?? null,
|
||||
segments: [serialize(s)],
|
||||
staged_at: new Date(),
|
||||
}))
|
||||
const { rows: result } = await retryDeadlock(() => db.rawQuery(
|
||||
`INSERT INTO staged_segments (file_key, counter_key, segments, staged_at)
|
||||
SELECT file_key, counter_key, segments, staged_at
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(file_key text, counter_key text, segments jsonb, staged_at timestamptz)
|
||||
ON CONFLICT (file_key) DO UPDATE SET segments = CASE
|
||||
-- The same article again (a crosspost, or a re-fetched range) is no second segment.
|
||||
WHEN EXISTS (SELECT 1 FROM jsonb_array_elements(staged_segments.segments) e
|
||||
WHERE e->>'messageId' = EXCLUDED.segments->0->>'messageId') THEN staged_segments.segments
|
||||
ELSE staged_segments.segments || EXCLUDED.segments END
|
||||
RETURNING file_key, jsonb_array_length(segments) AS n`,
|
||||
[JSON.stringify(rows)]
|
||||
))
|
||||
for (const row of result) if (Number(row.n) >= 2) promoted.push(row.file_key)
|
||||
}
|
||||
|
||||
// Counter files: two staged rows (two subjects) with one counter key make a file.
|
||||
const stagedCounters = [...new Set(toStage.map((s) => counterOf.get(s)).filter((k): k is string => Boolean(k)))]
|
||||
const readyCounters: string[] = []
|
||||
for (let i = 0; i < stagedCounters.length; i += CHUNK) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT counter_key FROM staged_segments WHERE counter_key = ANY(?) GROUP BY counter_key HAVING count(*) >= 2`,
|
||||
[stagedCounters.slice(i, i + CHUNK)]
|
||||
)
|
||||
for (const row of rows) readyCounters.push(row.counter_key)
|
||||
}
|
||||
|
||||
// Promoted files, staged segments of files this batch ingests anyway, and ready counter files.
|
||||
const fileKeys = [...new Set([...promoted, ...ingest.map((s) => s.fileKey)])]
|
||||
const counterKeys = [...new Set([...readyCounters, ...knownCounters])]
|
||||
const seen = new Set(ingest.map((s) => s.messageId))
|
||||
for (const segment of await this.take(fileKeys, counterKeys)) {
|
||||
if (!seen.has(segment.messageId)) ingest.push(segment)
|
||||
seen.add(segment.messageId)
|
||||
}
|
||||
const staged = toStage.filter((s) => !seen.has(s.messageId)).length
|
||||
return { ingest, staged }
|
||||
}
|
||||
|
||||
/**
|
||||
* GC: drops staged segments that waited longer than `maxAgeSeconds`, then
|
||||
* ingests those whose file got a binary meanwhile (two fetch jobs racing).
|
||||
* Expiry comes first so the table stays bounded whatever the budget; the
|
||||
* adoption starts from recently seen binaries (index on last_seen_at,
|
||||
* which moves in 10-minute steps) rather than joining every staged row.
|
||||
*/
|
||||
async sweep(maxAgeSeconds: number, budget: number): Promise<{ adopted: number; expired: number }> {
|
||||
let adopted = 0
|
||||
let expired = 0
|
||||
while (Date.now() < budget) {
|
||||
const result = await db.rawQuery(
|
||||
`DELETE FROM staged_segments WHERE ctid IN (
|
||||
SELECT ctid FROM staged_segments WHERE staged_at < now() - make_interval(secs => ?) LIMIT 20000)`,
|
||||
[maxAgeSeconds]
|
||||
)
|
||||
expired += result.rowCount
|
||||
if (result.rowCount < 20000) break
|
||||
}
|
||||
let lastId = 0
|
||||
while (Date.now() < budget) {
|
||||
const recent = await db
|
||||
.from('binaries')
|
||||
.where('last_seen_at', '>', new Date(Date.now() - (maxAgeSeconds + 15 * 60) * 1000))
|
||||
.where('id', '>', lastId)
|
||||
.orderBy('id')
|
||||
.limit(CHUNK)
|
||||
.select('id', 'file_key')
|
||||
if (recent.length === 0) break
|
||||
lastId = Number(recent[recent.length - 1].id)
|
||||
const keys = recent.map((b) => b.file_key as string)
|
||||
// A binary's key is a subject key or a counter key; staged rows are looked up by both.
|
||||
const segments = await this.take(keys, keys)
|
||||
if (segments.length > 0) await IngestService.ingest(segments)
|
||||
adopted += segments.length
|
||||
}
|
||||
return { adopted, expired }
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes and returns the staged segments of these files: by subject key,
|
||||
* or by counter key (returned as segments of the counter file).
|
||||
*/
|
||||
private async take(fileKeys: string[], counterKeys: string[] = []): Promise<IngestSegment[]> {
|
||||
const taken: IngestSegment[] = []
|
||||
for (let i = 0; i < fileKeys.length; i += CHUNK) {
|
||||
const { rows } = await retryDeadlock(() =>
|
||||
db.rawQuery(`DELETE FROM staged_segments WHERE file_key = ANY(?) RETURNING segments`, [fileKeys.slice(i, i + CHUNK)])
|
||||
)
|
||||
for (const row of rows) taken.push(...(row.segments as any[]).map(deserialize))
|
||||
}
|
||||
for (let i = 0; i < counterKeys.length; i += CHUNK) {
|
||||
const { rows } = await retryDeadlock(() =>
|
||||
db.rawQuery(`DELETE FROM staged_segments WHERE counter_key = ANY(?) RETURNING counter_key, segments`, [counterKeys.slice(i, i + CHUNK)])
|
||||
)
|
||||
for (const row of rows) taken.push(...(row.segments as any[]).map((s) => asCounterFile(deserialize(s), row.counter_key)))
|
||||
}
|
||||
return taken
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A segment as part of its counter file (D36): the counter key is its file
|
||||
* key, it is a weak identity (a repeated part number from another upload
|
||||
* marks the binary conflicted), and its release is a weak one of its own,
|
||||
* which the upload's par2 set can gather later.
|
||||
*/
|
||||
function asCounterFile(segment: IngestSegment, counterKey: string): IngestSegment {
|
||||
return {
|
||||
...segment,
|
||||
fileKey: counterKey,
|
||||
weakIdentity: true,
|
||||
release: {
|
||||
key: createHash('sha1').update(`counter-release\n${counterKey}`).digest('hex'),
|
||||
name: fileStem(segment.name),
|
||||
source: 'weak-stem',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a staging statement again when it lost a deadlock: concurrent fetch
|
||||
* jobs insert by file key and delete by counter key, so their row locks can
|
||||
* cross. Postgres rolled the statement back, and each one is safe to repeat.
|
||||
*/
|
||||
async function retryDeadlock<T>(statement: () => Promise<T>, attempts = 5): Promise<T> {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
return await statement()
|
||||
} catch (error: any) {
|
||||
if (error?.code !== '40P01' || attempt >= attempts) throw error
|
||||
await new Promise((resolve) => setTimeout(resolve, 50 * attempt + Math.random() * 100))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function serialize(segment: IngestSegment) {
|
||||
return {
|
||||
...segment,
|
||||
postedAt: segment.postedAt?.toISOString(),
|
||||
articleNumber: segment.articleNumber?.toString(),
|
||||
}
|
||||
}
|
||||
|
||||
function deserialize(stored: any): IngestSegment {
|
||||
return {
|
||||
...stored,
|
||||
postedAt: stored.postedAt ? new Date(stored.postedAt) : undefined,
|
||||
articleNumber: stored.articleNumber ?? undefined,
|
||||
}
|
||||
}
|
||||
|
||||
export default new StagingService()
|
||||
@@ -1,434 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { pruneSevenZipTails } from '#services/seven_zip_tails'
|
||||
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
|
||||
import type { EdgeBatch, ProbeCandidate, ReleaseChunk } from '#lib/edge/index'
|
||||
import ChunkPartitions from '#services/chunk_partitions'
|
||||
import CatalogRefs, { groupSetKey, posterText } from '#services/catalog_refs'
|
||||
import { decodeChunk } from '#lib/chunks/index'
|
||||
import { SIDECAR } from '#services/sidecar_folder'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
import { sourceCode } from '#lib/release/source'
|
||||
import { uploadTool } from '#lib/rules/index'
|
||||
import { byEffort } from '#services/pending_ladder'
|
||||
import { TEXT_KINDS, TEXT_MAX, textsToStore } from '#lib/release/text_kind'
|
||||
|
||||
/**
|
||||
* A small new release (at most 3 files, under 1 MB) whose files are all sidecars (.nfo, .sfv, .nzb, …): the sidecar
|
||||
* fold's candidates (releases.sidecar_only). Its file list decoded here, small as it is.
|
||||
*/
|
||||
function sidecarOnly(r: ReleaseChunk): boolean | null {
|
||||
if (r.files > 3 || r.bytes >= 1_000_000) return null
|
||||
const files = decodeChunk(r.payload)
|
||||
return files.length > 0 && files.every((f) => SIDECAR.test(f.name))
|
||||
}
|
||||
|
||||
export class StaleBatch extends Error {}
|
||||
|
||||
export interface StoredBatch {
|
||||
/** Release chunks in the batch. */
|
||||
releases: number
|
||||
/** Releases the batch started. */
|
||||
newReleases: number
|
||||
/** Provisional (tail) releases the batch finished. */
|
||||
finalized: number
|
||||
candidates: number
|
||||
/** Unnamed releases written to the pending store instead of the catalog (decision f). */
|
||||
pending: number
|
||||
}
|
||||
|
||||
/** A chunk's slot in its batch: the key and the upload the edge numbered it with (ProbeCandidate points at it). */
|
||||
const slot = (key: string, upload: number) => `${key}\n${upload}`
|
||||
|
||||
/**
|
||||
* Stores one batch of a leased range (REBUILD.md, "Voyager 3: window indexing", 2 and 4) in one transaction: each
|
||||
* release chunk is a whole release the edge's window finished, written once (its poster and group set by id, its
|
||||
* keys as bytea, its name into release_texts), with one chunk of its files, its probe candidates, and the batch's
|
||||
* rule counts and samples; then the range's resume point moves. A batch that doesn't start at the resume point was
|
||||
* sent twice or too late, and is refused (StaleBatch), so the sums stay right.
|
||||
*
|
||||
* A tail release comes twice: provisionally (finished: false, settled_at null, its candidates waiting) and then
|
||||
* finished, under the same release key (the identity, the group and its first article: lib/release/keys.ts), with
|
||||
* the files not sent before (a chunk's counts are its own; the finish adds them up); the finish is the one update a
|
||||
* release gets from the header pass. A finished release sent again is left as it is.
|
||||
*
|
||||
* Interim, until blocks are leased (the next step): chunks, counts and samples are keyed by the range's block
|
||||
* (ranges.block_id), and a range without one gets a block of its own here.
|
||||
*/
|
||||
class BatchStore {
|
||||
async store(rangeId: number, nodeId: number, batch: EdgeBatch): Promise<StoredBatch> {
|
||||
await ChunkPartitions.ensure()
|
||||
return db.transaction(async (trx) => {
|
||||
// One batch of a range at a time, without holding the range's row for the whole transaction: the
|
||||
// prefetcher's and heartbeats' updates of it waited on it (0.5 s on average, NNTP connections idle;
|
||||
// LOG.md 2026-10-09 (13)). The resume point is checked again when it moves, at the end.
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('range:' || ?::text, 0))`, [
|
||||
rangeId,
|
||||
])
|
||||
const range = await trx.from('ranges').where('id', rangeId).first()
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
|
||||
throw new StaleBatch(`Range ${rangeId} is not leased to this node`)
|
||||
if (Number(range.next_id) !== batch.from)
|
||||
throw new StaleBatch(
|
||||
`Range ${rangeId} is at ${range.next_id}, the batch starts at ${batch.from}`
|
||||
)
|
||||
const blockId = await this.blockOf(trx, range)
|
||||
const result: StoredBatch = {
|
||||
releases: batch.releases.length,
|
||||
newReleases: 0,
|
||||
finalized: 0,
|
||||
candidates: 0,
|
||||
pending: 0,
|
||||
}
|
||||
|
||||
// Nothing enters the catalog without a usable name (decision f): an unnamed release (no readable file name)
|
||||
// goes to the pending store with its probe targets, the named ones to the catalog.
|
||||
const named = batch.releases.filter((r) => r.readable)
|
||||
const unnamed = batch.releases.filter((r) => !r.readable)
|
||||
const written = await this.writeReleases(
|
||||
trx,
|
||||
{ groupId: Number(range.group_id), blockId },
|
||||
batch.from,
|
||||
named,
|
||||
batch.candidates
|
||||
)
|
||||
result.newReleases = written.newReleases
|
||||
result.finalized = written.finalized
|
||||
result.candidates = written.candidates
|
||||
result.pending = await this.writePending(
|
||||
trx,
|
||||
Number(range.group_id),
|
||||
unnamed,
|
||||
batch.candidates
|
||||
)
|
||||
|
||||
if (batch.counts.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO rule_counts (block_id, rule, key, articles, files)
|
||||
SELECT ?, v.rule, v.key, v.articles, v.files FROM jsonb_to_recordset(?::jsonb) AS v(rule text, key text, articles bigint, files bigint)
|
||||
ON CONFLICT (block_id, rule, key) DO UPDATE SET articles = rule_counts.articles + excluded.articles, files = rule_counts.files + excluded.files`,
|
||||
[blockId, JSON.stringify(batch.counts)]
|
||||
)
|
||||
}
|
||||
|
||||
for (const s of batch.samples) {
|
||||
await trx.insertQuery().table('rule_samples').insert({
|
||||
block_id: blockId,
|
||||
rule: s.rule,
|
||||
rule_key: s.ruleKey,
|
||||
release_key: s.key,
|
||||
name: s.name,
|
||||
files: s.files,
|
||||
segments: s.segments,
|
||||
payload: s.payload,
|
||||
})
|
||||
}
|
||||
|
||||
const moved = await trx
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
.where('status', 'leased')
|
||||
.where('node_id', nodeId)
|
||||
.where('next_id', batch.from)
|
||||
.update({
|
||||
next_id: batch.next,
|
||||
headers: db.raw('headers + ?', [batch.headers]),
|
||||
// The range's posting time span: the batches' trimmed spans, min and max (least and greatest skip nulls).
|
||||
posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [
|
||||
batch.postedLow ?? null,
|
||||
]),
|
||||
posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [
|
||||
batch.postedHigh ?? null,
|
||||
]),
|
||||
heartbeat_at: new Date(),
|
||||
})
|
||||
if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1))
|
||||
throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
|
||||
return result
|
||||
})
|
||||
}
|
||||
|
||||
/** The range's block; a range without one (no planner yet) is a block of its own, leased as the range is. */
|
||||
async blockOf(trx: TransactionClientContract, range: any): Promise<number> {
|
||||
if (range.block_id) return Number(range.block_id)
|
||||
const [block] = await trx
|
||||
.table('blocks')
|
||||
.insert({
|
||||
group_id: range.group_id,
|
||||
start_id: range.start_id,
|
||||
end_id: range.end_id,
|
||||
next_id: range.next_id,
|
||||
status: 'leased',
|
||||
priority: range.priority ?? 0,
|
||||
node_id: range.node_id,
|
||||
leased_at: range.leased_at ?? new Date(),
|
||||
heartbeat_at: new Date(),
|
||||
})
|
||||
.returning('id')
|
||||
await trx.from('ranges').where('id', range.id).update({ block_id: block.id })
|
||||
return Number(block.id)
|
||||
}
|
||||
|
||||
/**
|
||||
* The unnamed releases of a batch into pending_releases (migration 6300, the HDD): keys, counts, the upload
|
||||
* tool and the probe targets, no chunk. The pending ladder (REBUILD.md decision f) probes them cheapest first,
|
||||
* and a name moves one into the catalog; a provisional tail release sent again adds its counts up.
|
||||
*/
|
||||
async writePending(
|
||||
trx: TransactionClientContract,
|
||||
groupId: number,
|
||||
releases: ReleaseChunk[],
|
||||
candidates: ProbeCandidate[]
|
||||
): Promise<number> {
|
||||
if (releases.length === 0) return 0
|
||||
const targets = new Map<
|
||||
string,
|
||||
{ kind: string; messageId: string; file: string; bytes?: number }[]
|
||||
>()
|
||||
for (const c of candidates) {
|
||||
const key = slot(c.releaseKey, c.upload ?? 0)
|
||||
const list = targets.get(key) ?? []
|
||||
list.push({ kind: c.kind, messageId: c.messageId, file: c.file, bytes: c.bytes })
|
||||
targets.set(key, list)
|
||||
}
|
||||
let written = 0
|
||||
for (let i = 0; i < releases.length; i += 1000) {
|
||||
const rows = releases.slice(i, i + 1000).map((r) => {
|
||||
const probes = targets.get(slot(r.key, r.upload)) ?? []
|
||||
return {
|
||||
release_key: releaseKey(r.key, groupId, r.firstArticle).toString('hex'),
|
||||
identity_key: identityKey(r.key).toString('hex'),
|
||||
name: r.name.slice(0, TEXT_MAX),
|
||||
poster: r.poster || null,
|
||||
first: r.postedAt,
|
||||
last: r.postedLast ?? r.postedAt,
|
||||
first_article: r.firstArticle,
|
||||
last_article: r.lastArticle,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
bytes: r.bytes,
|
||||
partial: r.partial === true,
|
||||
finished: r.finished !== false,
|
||||
file_total: r.fileTotal,
|
||||
tool: r.tool ?? (probes[0] ? uploadTool(probes[0].messageId) : 'unknown'),
|
||||
candidates: probes,
|
||||
// The first target's article (by effort): the cheapest probe first within a class (LOG.md (130)).
|
||||
target_bytes: byEffort(probes)[0]?.bytes ?? null,
|
||||
}
|
||||
})
|
||||
const { rows: done } = await trx.rawQuery(
|
||||
`INSERT INTO pending_releases (release_key, identity_key, group_id, name, poster, posted_at, last_posted_at, first_article, last_article,
|
||||
files, segments, bytes, partial, finished, file_total, tool, candidates, klass, target_bytes)
|
||||
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.name, v.poster, to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0),
|
||||
v.first_article, v.last_article, v.files, v.segments, v.bytes, v.partial, v.finished, v.file_total, v.tool, v.candidates,
|
||||
pending_class(v.name, v.bytes, v.partial), v.target_bytes
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, identity_key text, name text, poster text, first bigint, last bigint, first_article bigint, last_article bigint,
|
||||
files int, segments bigint, bytes bigint, partial boolean, finished boolean, file_total int, tool text, candidates jsonb, target_bytes bigint)
|
||||
ON CONFLICT (release_key) DO UPDATE SET
|
||||
files = pending_releases.files + excluded.files, segments = pending_releases.segments + excluded.segments, bytes = pending_releases.bytes + excluded.bytes,
|
||||
last_posted_at = greatest(pending_releases.last_posted_at, excluded.last_posted_at), last_article = greatest(pending_releases.last_article, excluded.last_article),
|
||||
partial = excluded.partial, finished = excluded.finished, file_total = coalesce(excluded.file_total, pending_releases.file_total),
|
||||
candidates = CASE WHEN jsonb_array_length(pending_releases.candidates) = 0 THEN excluded.candidates ELSE pending_releases.candidates END,
|
||||
klass = pending_class(pending_releases.name, pending_releases.bytes + excluded.bytes, excluded.partial),
|
||||
target_bytes = coalesce(pending_releases.target_bytes, excluded.target_bytes)
|
||||
WHERE NOT pending_releases.finished
|
||||
RETURNING release_key, tool, klass, (xmax = 0) AS inserted`,
|
||||
[groupId, JSON.stringify(rows)]
|
||||
)
|
||||
written += done.length
|
||||
// Each class seen gets its row in the ladder's record, so the claim explores it (pending_classes).
|
||||
const classes = new Map<string, { tool: string; klass: number; waiting: number }>()
|
||||
for (const d of done) {
|
||||
const key = `${d.tool}\t${d.klass}`
|
||||
const entry = classes.get(key) ?? { tool: d.tool, klass: Number(d.klass), waiting: 0 }
|
||||
if (d.inserted) entry.waiting++
|
||||
classes.set(key, entry)
|
||||
}
|
||||
if (classes.size > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO pending_classes (tool, group_id, klass, waiting)
|
||||
SELECT v.tool, ?, v.klass, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, klass int, waiting int)
|
||||
ON CONFLICT (tool, group_id, klass) DO UPDATE SET waiting = pending_classes.waiting + excluded.waiting`,
|
||||
[groupId, JSON.stringify([...classes.values()])]
|
||||
)
|
||||
}
|
||||
}
|
||||
return written
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases of a batch into the catalog, one chunk each, and their probe candidates. Every chunk is one
|
||||
* release: inserted, or (a provisional tail release sent again) finished.
|
||||
*/
|
||||
async writeReleases(
|
||||
trx: TransactionClientContract,
|
||||
where: { groupId: number; blockId: number },
|
||||
batchFrom: number,
|
||||
releases: ReleaseChunk[],
|
||||
candidates: ProbeCandidate[]
|
||||
) {
|
||||
const out = { newReleases: 0, finalized: 0, candidates: 0 }
|
||||
if (releases.length === 0) return out
|
||||
/** Release ids by slot, for the chunks and candidates; a slot not here was a finished release sent again. */
|
||||
const ids = new Map<string, number>()
|
||||
const finalized = new Set<number>()
|
||||
|
||||
const posters = await CatalogRefs.posterIds(trx, releases.map((r) => r.poster).filter(Boolean))
|
||||
const groupSets = await CatalogRefs.groupSetIds(
|
||||
trx,
|
||||
releases.map((r) => r.groups)
|
||||
)
|
||||
const sorted = [...releases].sort((a, b) => a.key.localeCompare(b.key) || a.upload - b.upload)
|
||||
for (let i = 0; i < sorted.length; i += 1000) {
|
||||
const part = sorted.slice(i, i + 1000)
|
||||
const rows = part.map((r, idx) => ({
|
||||
idx,
|
||||
release_key: releaseKey(r.key, where.groupId, r.firstArticle).toString('hex'),
|
||||
identity_key: identityKey(r.key).toString('hex'),
|
||||
name: r.name,
|
||||
source: sourceCode(r.source),
|
||||
poster_id: r.poster ? (posters.get(posterText(r.poster)) ?? null) : null,
|
||||
group_set_id: groupSets.get(groupSetKey(r.groups))!,
|
||||
first: r.postedAt,
|
||||
last: r.postedLast ?? r.postedAt,
|
||||
file_total: r.fileTotal,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
bytes: r.bytes,
|
||||
first_article: r.firstArticle,
|
||||
last_article: r.lastArticle,
|
||||
readable: r.readable,
|
||||
has_par2: r.hasPar2,
|
||||
finished: r.finished !== false,
|
||||
edge: r.edge === true,
|
||||
partial: r.partial === true,
|
||||
sidecar_only: sidecarOnly(r),
|
||||
}))
|
||||
// Inserted, or a provisional release of the key finished (the chunk's files added up); a finished release
|
||||
// sent again (no row returned) is left as it is.
|
||||
const { rows: written } = await trx.rawQuery(
|
||||
`INSERT INTO releases (release_key, identity_key, group_id, group_set_id, poster_id, name, source, posted_at, last_posted_at, file_total, files, segments, bytes, chunks,
|
||||
first_article, last_article, readable, has_par2, complete, edge, sidecar_only, settled_at)
|
||||
SELECT decode(v.release_key, 'hex'), decode(v.identity_key, 'hex'), ?, v.group_set_id, v.poster_id, v.name, v.source,
|
||||
to_timestamp(v.first / 1000.0), to_timestamp(v.last / 1000.0), v.file_total, v.files, v.segments, v.bytes, 1,
|
||||
v.first_article, v.last_article, v.readable, v.has_par2,
|
||||
CASE WHEN v.partial THEN false WHEN v.file_total IS NULL THEN NULL ELSE v.files >= v.file_total END,
|
||||
v.edge, v.sidecar_only, CASE WHEN v.finished THEN now() END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(idx int, release_key text, identity_key text, name text, source int, poster_id int, group_set_id int, first bigint, last bigint,
|
||||
file_total int, files int, segments bigint, bytes bigint, first_article bigint, last_article bigint, readable boolean, has_par2 boolean,
|
||||
finished boolean, edge boolean, partial boolean, sidecar_only boolean)
|
||||
ON CONFLICT (release_key) DO UPDATE SET
|
||||
files = releases.files + excluded.files, segments = releases.segments + excluded.segments, bytes = releases.bytes + excluded.bytes,
|
||||
chunks = releases.chunks + CASE WHEN excluded.files > 0 THEN 1 ELSE 0 END,
|
||||
posted_at = least(releases.posted_at, excluded.posted_at), last_posted_at = greatest(releases.last_posted_at, excluded.last_posted_at),
|
||||
file_total = coalesce(excluded.file_total, releases.file_total),
|
||||
first_article = least(releases.first_article, excluded.first_article), last_article = greatest(releases.last_article, excluded.last_article),
|
||||
readable = releases.readable OR excluded.readable, has_par2 = releases.has_par2 OR excluded.has_par2,
|
||||
complete = CASE WHEN releases.complete IS FALSE AND releases.file_total IS NULL THEN false
|
||||
WHEN coalesce(excluded.file_total, releases.file_total) IS NULL THEN releases.complete
|
||||
ELSE releases.files + excluded.files >= coalesce(excluded.file_total, releases.file_total) END,
|
||||
edge = excluded.edge, sidecar_only = coalesce(releases.sidecar_only, excluded.sidecar_only), settled_at = excluded.settled_at
|
||||
WHERE releases.settled_at IS NULL
|
||||
RETURNING id, encode(release_key, 'hex') AS release_key, (xmax = 0) AS inserted, settled_at`,
|
||||
[where.groupId, JSON.stringify(rows)]
|
||||
)
|
||||
const byKey = new Map<string, { id: number; inserted: boolean; settled: boolean }>(
|
||||
written.map(
|
||||
(w: { id: string; release_key: string; inserted: boolean; settled_at: Date | null }) => [
|
||||
w.release_key,
|
||||
{ id: Number(w.id), inserted: w.inserted, settled: w.settled_at !== null },
|
||||
]
|
||||
)
|
||||
)
|
||||
const texts: { release_id: number; kind: number; text: string }[] = []
|
||||
for (const row of rows) {
|
||||
const w = byKey.get(row.release_key)
|
||||
if (!w) continue
|
||||
const r = part[row.idx]
|
||||
ids.set(slot(r.key, r.upload), w.id)
|
||||
if (w.inserted) out.newReleases++
|
||||
else if (w.settled) {
|
||||
out.finalized++
|
||||
finalized.add(w.id)
|
||||
}
|
||||
texts.push({ release_id: w.id, kind: TEXT_KINDS.name, text: row.name.slice(0, TEXT_MAX) })
|
||||
// An album's members stay findable by their own names (the first TEXTS_PER_KIND of them).
|
||||
if (r.source === 'album' && r.files > 0) {
|
||||
for (const text of textsToStore(decodeChunk(r.payload).map((f) => f.name))) {
|
||||
texts.push({ release_id: w.id, kind: TEXT_KINDS.member, text })
|
||||
}
|
||||
}
|
||||
}
|
||||
// The name (and an album's members), searchable (release_texts; a text sent again is a no-op).
|
||||
if (texts.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO release_texts (release_id, kind, text)
|
||||
SELECT v.release_id, v.kind, v.text FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, kind int, text text) WHERE v.text <> ''
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[JSON.stringify(texts)]
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// One write-once chunk per release (a finish adds the files not sent before).
|
||||
for (let i = 0; i < sorted.length; i += 500) {
|
||||
const rows = sorted
|
||||
.slice(i, i + 500)
|
||||
.filter((r) => ids.has(slot(r.key, r.upload)) && r.files > 0)
|
||||
.map((r) => ({
|
||||
release_id: ids.get(slot(r.key, r.upload))!,
|
||||
block_id: where.blockId,
|
||||
batch_from: batchFrom,
|
||||
files: r.files,
|
||||
segments: r.segments,
|
||||
payload: r.payload,
|
||||
}))
|
||||
if (rows.length > 0) await trx.insertQuery().table('release_chunks').multiInsert(rows)
|
||||
}
|
||||
|
||||
// A finished release's candidates are claimable at once; a provisional one's wait for its finish.
|
||||
const settled = new Set<number>()
|
||||
for (const r of sorted) {
|
||||
const id = ids.get(slot(r.key, r.upload))
|
||||
if (id !== undefined && r.finished !== false) settled.add(id)
|
||||
}
|
||||
const chunks = new Map(sorted.map((r) => [slot(r.key, r.upload), r]))
|
||||
const kept = candidates.filter((c) => ids.has(slot(c.releaseKey, c.upload ?? 0)))
|
||||
for (let i = 0; i < kept.length; i += 2000) {
|
||||
const rows = kept.slice(i, i + 2000).map((c) => {
|
||||
const releaseId = ids.get(slot(c.releaseKey, c.upload ?? 0))!
|
||||
const chunk = chunks.get(slot(c.releaseKey, c.upload ?? 0))!
|
||||
return {
|
||||
release_id: releaseId,
|
||||
kind: c.kind,
|
||||
message_id: c.messageId,
|
||||
file: c.file,
|
||||
ready: settled.has(releaseId),
|
||||
unreadable: !chunk.readable,
|
||||
}
|
||||
})
|
||||
const added = await trx
|
||||
.knexQuery()
|
||||
.table('probe_candidates')
|
||||
.insert(rows)
|
||||
.onConflict(['release_id', 'kind', 'message_id'])
|
||||
.ignore()
|
||||
.returning('id')
|
||||
out.candidates += added.length
|
||||
// A later piece of a split 7z: the tails of lower volumes go (app/services/seven_zip_tails.ts).
|
||||
const tails = new Set(rows.filter((r) => r.kind === '7z-tail').map((r) => r.release_id))
|
||||
if (added.length > 0 && tails.size > 0) await pruneSevenZipTails(trx, [...tails])
|
||||
}
|
||||
// The finish of a provisional release: its waiting candidates become claimable.
|
||||
if (finalized.size > 0) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE pc.release_id = ANY(?) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||
[[...finalized]]
|
||||
)
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
}
|
||||
|
||||
export default new BatchStore()
|
||||
@@ -1,93 +0,0 @@
|
||||
import type db from '@adonisjs/lucid/services/db'
|
||||
|
||||
type Client = Pick<typeof db, 'rawQuery'>
|
||||
|
||||
/** Posters are kept to this length (as the trigram index and poster search always did; a btree entry holds 2,704 bytes). */
|
||||
export const POSTER_MAX = 200
|
||||
/** Cached ids per process: group sets are ~2,600 in all; posters many more, so the cache is emptied when it grows past this. */
|
||||
const CACHE_MAX = 200_000
|
||||
|
||||
const posterCache = new Map<string, number>()
|
||||
const groupSetCache = new Map<string, number>()
|
||||
|
||||
/** A group list as group_sets stores it: sorted, without repeats, as JSON. */
|
||||
export function groupSetKey(groups: string[]): string {
|
||||
return JSON.stringify([...new Set(groups)].sort())
|
||||
}
|
||||
|
||||
/** A poster as posters stores it. */
|
||||
export function posterText(poster: string): string {
|
||||
return poster.slice(0, POSTER_MAX)
|
||||
}
|
||||
|
||||
async function ids(
|
||||
client: Client,
|
||||
cache: Map<string, number>,
|
||||
table: 'posters' | 'group_sets',
|
||||
column: 'poster' | 'groups',
|
||||
cast: 'text' | 'jsonb',
|
||||
values: string[]
|
||||
): Promise<Map<string, number>> {
|
||||
const out = new Map<string, number>()
|
||||
const missing: string[] = []
|
||||
for (const v of values) {
|
||||
const id = cache.get(v)
|
||||
if (id === undefined) missing.push(v)
|
||||
else out.set(v, id)
|
||||
}
|
||||
if (missing.length === 0) return out
|
||||
// Insert what's new, then read every id (ON CONFLICT DO NOTHING returns nothing for the rows that existed).
|
||||
await client.rawQuery(
|
||||
`INSERT INTO ${table} (${column}) SELECT DISTINCT v::${cast} FROM unnest(?::text[]) AS v ON CONFLICT DO NOTHING`,
|
||||
[missing]
|
||||
)
|
||||
const { rows } = await client.rawQuery(
|
||||
`SELECT id, ${column}::text AS value FROM ${table} WHERE ${column} = ANY(?::${cast}[])`,
|
||||
[missing]
|
||||
)
|
||||
if (cache.size + rows.length > CACHE_MAX) cache.clear()
|
||||
for (const r of rows) {
|
||||
// jsonb comes back in Postgres' spelling ("["a", "b"]"); the caller's key is the JSON we sent.
|
||||
const key = cast === 'jsonb' ? JSON.stringify(JSON.parse(r.value)) : r.value
|
||||
cache.set(key, Number(r.id))
|
||||
out.set(key, Number(r.id))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Ids in the catalog's lookup tables (voyager 3, migration 6000): a release's poster_id and group_set_id. New
|
||||
* values are inserted; known ones come from a cache per process (the batch store sees the same posters and
|
||||
* group sets batch after batch).
|
||||
*/
|
||||
export default {
|
||||
/** Ids by poster (posterText applied), for a batch of posters. */
|
||||
async posterIds(client: Client, posters: string[]): Promise<Map<string, number>> {
|
||||
return ids(client, posterCache, 'posters', 'poster', 'text', [
|
||||
...new Set(posters.map(posterText)),
|
||||
])
|
||||
},
|
||||
|
||||
async posterId(client: Client, poster: string): Promise<number> {
|
||||
const found = await this.posterIds(client, [poster])
|
||||
return found.get(posterText(poster))!
|
||||
},
|
||||
|
||||
/** Ids by groupSetKey, for a batch of group lists. */
|
||||
async groupSetIds(client: Client, sets: string[][]): Promise<Map<string, number>> {
|
||||
return ids(client, groupSetCache, 'group_sets', 'groups', 'jsonb', [
|
||||
...new Set(sets.map(groupSetKey)),
|
||||
])
|
||||
},
|
||||
|
||||
async groupSetId(client: Client, groups: string[]): Promise<number> {
|
||||
const found = await this.groupSetIds(client, [groups])
|
||||
return found.get(groupSetKey(groups))!
|
||||
},
|
||||
|
||||
/** Forgets the cached ids (tests roll their transactions back, so the ids they saw are gone). */
|
||||
forget() {
|
||||
posterCache.clear()
|
||||
groupSetCache.clear()
|
||||
},
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
/** How often the catalog's size is measured (pg_tablespace_size walks its files). */
|
||||
const CHECK_MS = 60_000
|
||||
|
||||
/**
|
||||
* The catalog's room on the NVMe (CATALOG_MAX_NVME_GB, like voyager 1's
|
||||
* BACKFILL_MAX_NVME_GB): once its tablespace (the database's
|
||||
* default_tablespace) is over the cap, only the tail and the stored ranges
|
||||
* (priority below 0) are leased and prefetched, and history waits. After
|
||||
* reset 24 the catalog grew ~3.7 GB an hour, root down to 31 GB free with
|
||||
* 600k history ranges to go (LOG.md 2026-10-14 (53)). Off when unset.
|
||||
*/
|
||||
class CatalogSpace {
|
||||
private checkedAt = 0
|
||||
private over = false
|
||||
|
||||
async full(): Promise<boolean> {
|
||||
const cap = coordinatorConfig.catalogMaxNvmeGb
|
||||
if (!cap) return false
|
||||
if (Date.now() - this.checkedAt < CHECK_MS) return this.over
|
||||
this.checkedAt = Date.now()
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
|
||||
)
|
||||
const bytes = Number(rows[0].bytes)
|
||||
const was = this.over
|
||||
this.over = bytes > cap * 1e9
|
||||
if (this.over !== was) {
|
||||
logger.info(`Catalog ${(bytes / 1e9).toFixed(1)} GB of ${cap}: history ${this.over ? 'paused (tail and stored ranges only)' : 'resumed'}.`)
|
||||
}
|
||||
return this.over
|
||||
}
|
||||
}
|
||||
|
||||
export default new CatalogSpace()
|
||||
@@ -1,30 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import env from '#start/env'
|
||||
|
||||
/** Months ahead that always have a release_chunks partition. */
|
||||
const AHEAD = 2
|
||||
|
||||
/**
|
||||
* release_chunks is partitioned by month of writing (REBUILD.md, decision a).
|
||||
* Inserts fail without a partition for now(), so the current and the next
|
||||
* months' partitions are created before they are needed.
|
||||
*/
|
||||
class ChunkPartitions {
|
||||
async ensure(now = new Date()) {
|
||||
// Checked each time, not remembered: a rolled-back transaction can take a created partition along.
|
||||
const current = `release_chunks_${now.toISOString().slice(0, 7).replace('-', '')}`
|
||||
const { rows } = await db.rawQuery(`SELECT to_regclass(?) IS NOT NULL AS present`, [current])
|
||||
if (rows[0].present) return
|
||||
const tablespace = env.get('CHUNK_TABLESPACE')
|
||||
for (let i = 0; i <= AHEAD; i++) {
|
||||
const from = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + i, 1))
|
||||
const to = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth() + i + 1, 1))
|
||||
const name = `release_chunks_${from.toISOString().slice(0, 7).replace('-', '')}`
|
||||
await db.rawQuery(
|
||||
`CREATE TABLE IF NOT EXISTS ${name} PARTITION OF release_chunks FOR VALUES FROM ('${from.toISOString()}') TO ('${to.toISOString()}')${tablespace ? ` TABLESPACE ${tablespace}` : ''}`
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new ChunkPartitions()
|
||||
@@ -1,120 +0,0 @@
|
||||
import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import { deflate as deflateCb, gzip as gzipCb, unzip } from 'node:zlib'
|
||||
import { promisify } from 'node:util'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { filterOverviews } from '#lib/edge/index'
|
||||
import type { ChunkSpan } from '#services/spool'
|
||||
|
||||
const inflate = promisify(unzip)
|
||||
const deflate = promisify(deflateCb)
|
||||
const gzip = promisify(gzipCb)
|
||||
/** The largest inflated chunk the filter reads as one string (V8's limit is 0x1fffffe8 characters). */
|
||||
const MAX_FILTER_BYTES = 256 * 1024 * 1024
|
||||
|
||||
/** What a filtered chunk was filtered with (CORPUS_FILTER): the rule set and the group priorities of its lease. */
|
||||
export interface CorpusFilter {
|
||||
ruleVersion: number
|
||||
priorities: Record<string, number>
|
||||
}
|
||||
/** The rules a filtered chunk leaves out (lib/edge/corpus_filter.ts), in its range.json. */
|
||||
export const FILTERED = ['crosspost', 'obfuscated']
|
||||
|
||||
export interface CorpusRange {
|
||||
group: string
|
||||
start: number
|
||||
end: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Stored XOVER chunks of the ranges marked `capture` (REBUILD.md, corpus):
|
||||
* `<dir>/<group>/<start>-<end>/<nnnn>-<from>-<to>.gz` with a range.json once
|
||||
* every chunk is there, the layout edge-bench reads (edge/fixtures). The
|
||||
* grouping sweeps replay them with each build's rules instead of waiting for
|
||||
* a re-index; the prefetcher reads a stored chunk instead of fetching it
|
||||
* again. Off without CORPUS_DIR. Filtered (CORPUS_FILTER; LOG.md 2026-10-23
|
||||
* (101)): the lines the edge worker only counts are left out, ~half of them,
|
||||
* and each chunk's `.drops` beside it (gzip JSON, ChunkDrops) lists their
|
||||
* article numbers by rule; its range.json says `filtered`.
|
||||
*/
|
||||
class Corpus {
|
||||
get dir(): string | undefined {
|
||||
return coordinatorConfig.corpus.dir
|
||||
}
|
||||
|
||||
private rangeDir(range: CorpusRange, dir = this.dir!) {
|
||||
return join(dir, range.group, `${range.start}-${range.end}`)
|
||||
}
|
||||
|
||||
private path(range: CorpusRange, span: ChunkSpan, dir = this.dir!) {
|
||||
const index = Math.floor((span.from - range.start) / coordinatorConfig.chunkArticles)
|
||||
return join(this.rangeDir(range, dir), `${String(index).padStart(4, '0')}-${span.from}-${span.to}.gz`)
|
||||
}
|
||||
|
||||
/** Whether every chunk of the range is stored (its range.json). */
|
||||
async complete(range: CorpusRange): Promise<boolean> {
|
||||
if (!this.dir) return false
|
||||
return access(join(this.rangeDir(range), 'range.json')).then(
|
||||
() => true,
|
||||
() => false
|
||||
)
|
||||
}
|
||||
|
||||
/** Whether one chunk is stored. */
|
||||
async has(range: CorpusRange, span: ChunkSpan): Promise<boolean> {
|
||||
if (!this.dir) return false
|
||||
return access(this.path(range, span)).then(
|
||||
() => true,
|
||||
() => false
|
||||
)
|
||||
}
|
||||
|
||||
/** A stored chunk, or null (`dir`: another corpus than CORPUS_DIR's, the namer's tests). */
|
||||
async read(range: CorpusRange, span: ChunkSpan, dir = this.dir): Promise<Buffer | null> {
|
||||
if (!dir) return null
|
||||
return readFile(this.path(range, span, dir)).catch(() => null)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores a chunk that inflates (a reply cut short isn't kept), and the
|
||||
* range.json once the range is complete (`total` chunks). With `filter`,
|
||||
* without the lines the edge worker drops on their own, compressed again
|
||||
* as the news server's (zlib), with its `.drops` written first.
|
||||
*/
|
||||
async write(range: CorpusRange, span: ChunkSpan, data: Buffer, total: number, filter?: CorpusFilter) {
|
||||
if (!this.dir) return
|
||||
let raw: Buffer
|
||||
try {
|
||||
raw = await inflate(data)
|
||||
} catch {
|
||||
return
|
||||
}
|
||||
const dir = this.rangeDir(range)
|
||||
const file = this.path(range, span)
|
||||
await mkdir(dir, { recursive: true })
|
||||
let stored = data
|
||||
// Past V8's longest string (0x1fffffe8) the text can't be filtered; such a chunk is stored as it came (the
|
||||
// worker drops the same lines itself). Seen once: a chunk inflating past 512 MB (LOG.md 2026-10-24 (102)).
|
||||
if (filter && raw.length < MAX_FILTER_BYTES) {
|
||||
// Latin-1 both ways: every byte back as it was.
|
||||
const { kept, drops } = filterOverviews(raw.toString('latin1'), range.group, filter.priorities, filter.ruleVersion)
|
||||
stored = await deflate(Buffer.from(kept, 'latin1'), { level: 9 })
|
||||
await writeFile(`${file}.drops.part`, await gzip(JSON.stringify(drops)))
|
||||
await rename(`${file}.drops.part`, `${file}.drops`)
|
||||
}
|
||||
await writeFile(`${file}.part`, stored)
|
||||
await rename(`${file}.part`, file)
|
||||
const chunks = (await readdir(dir)).filter((f) => f.endsWith('.gz')).length
|
||||
if (chunks >= total) await writeFile(join(dir, 'range.json'), JSON.stringify(filter ? { ...range, filtered: FILTERED } : range))
|
||||
}
|
||||
|
||||
/** A chunk a worker couldn't inflate after all: not served again, and the range not complete. */
|
||||
async removeChunk(range: CorpusRange, span: ChunkSpan) {
|
||||
if (!this.dir) return
|
||||
await rm(this.path(range, span), { force: true })
|
||||
await rm(`${this.path(range, span)}.drops`, { force: true })
|
||||
await rm(join(this.rangeDir(range), 'range.json'), { force: true })
|
||||
}
|
||||
}
|
||||
|
||||
export default new Corpus()
|
||||
@@ -1,40 +0,0 @@
|
||||
import { statfs } from 'node:fs/promises'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/** How often free space is measured. */
|
||||
const CHECK_MS = 60_000
|
||||
/** Resumed only this much above the floor: not paused and resumed at every flush. */
|
||||
const MARGIN_GB = 1
|
||||
|
||||
/**
|
||||
* Free space on the filesystem holding `path`, against a floor: low below it, again fine MARGIN_GB above it. In a
|
||||
* container `/` is an overlay on the host's Docker root, which on homelab is the NVMe the catalog and the WAL are on.
|
||||
* The prober filled it ~0.8–1.1 GB an hour at ~200 articles/s, and nothing stopped it at the 10 GB floor (LOG.md
|
||||
* 2026-10-18 (77)). Off when the floor is 0.
|
||||
*/
|
||||
export class DiskSpace {
|
||||
private checkedAt = 0
|
||||
private low = false
|
||||
|
||||
constructor(
|
||||
private what: string,
|
||||
private path: string,
|
||||
private floorGb: number,
|
||||
private free: (path: string) => Promise<number> = async (p) => {
|
||||
const s = await statfs(p)
|
||||
return s.bavail * s.bsize
|
||||
},
|
||||
private checkMs = CHECK_MS
|
||||
) {}
|
||||
|
||||
async short(): Promise<boolean> {
|
||||
if (!this.floorGb) return false
|
||||
if (Date.now() - this.checkedAt < this.checkMs) return this.low
|
||||
this.checkedAt = Date.now()
|
||||
const gb = (await this.free(this.path)) / 1e9
|
||||
const was = this.low
|
||||
this.low = gb < this.floorGb + (was ? MARGIN_GB : 0)
|
||||
if (this.low !== was) logger.info(`${gb.toFixed(1)} GB free (floor ${this.floorGb} GB): ${this.what} ${this.low ? 'paused' : 'resumed'}.`)
|
||||
return this.low
|
||||
}
|
||||
}
|
||||
@@ -1,138 +0,0 @@
|
||||
import type { Overview, XoverClient } from '#lib/nntp/index'
|
||||
|
||||
/** When an article of a group was posted: the median Date of a slice of articles at `article`. */
|
||||
export interface DateSample {
|
||||
article: number
|
||||
postedAt: Date
|
||||
}
|
||||
|
||||
/** Before this, a Date header is a broken clock (the oldest retention is 2008). */
|
||||
const OLDEST = Date.UTC(1995, 0, 1)
|
||||
const DAY = 86_400_000
|
||||
|
||||
/**
|
||||
* The median of the overviews' Date headers that parse and are plausible (1995 … a day from `now`), with at least
|
||||
* `min` of them; null otherwise (an expired or empty slice).
|
||||
*/
|
||||
export function medianDate(overviews: Overview[], now = new Date(), min = 3): Date | null {
|
||||
const latest = now.getTime() + DAY
|
||||
const times = overviews
|
||||
.map(([, fields]) => Date.parse(fields.date ?? ''))
|
||||
.filter((t) => t >= OLDEST && t <= latest)
|
||||
.sort((a, b) => a - b)
|
||||
if (times.length < min) return null
|
||||
return new Date(times[Math.floor(times.length / 2)])
|
||||
}
|
||||
|
||||
/** Where a group is sampled: `count` article numbers spread over low … high, the first at low and the last `size` before high. */
|
||||
export function samplePoints(low: number, high: number, count: number, size: number): number[] {
|
||||
const last = Math.max(low, high - size + 1)
|
||||
if (count <= 1 || last <= low) return [low]
|
||||
const points = new Set<number>()
|
||||
for (let i = 0; i < count; i++) points.add(Math.round(low + ((last - low) * i) / (count - 1)))
|
||||
return [...points].sort((a, b) => a - b)
|
||||
}
|
||||
|
||||
/**
|
||||
* The samples as a posting-time curve: in article order, a sample later than both of the next two by over a day
|
||||
* (a slice of one poster's wrong clock) is dropped, then each date is at least its predecessor's (articles arrive
|
||||
* in number order, bar stragglers and old crossposts).
|
||||
*/
|
||||
export function monotone(samples: DateSample[]): DateSample[] {
|
||||
const sorted = [...samples].sort((a, b) => a.article - b.article)
|
||||
const kept = sorted.filter((s, i) => {
|
||||
const next = sorted.slice(i + 1, i + 3)
|
||||
return next.length < 2 || next.some((n) => n.postedAt.getTime() + DAY >= s.postedAt.getTime())
|
||||
})
|
||||
let floor = Number.NEGATIVE_INFINITY
|
||||
return kept.map((s) => {
|
||||
floor = Math.max(floor, s.postedAt.getTime())
|
||||
return { article: s.article, postedAt: new Date(floor) }
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* When `article` was posted by the group's samples (monotone, in article order): interpolated between the two around
|
||||
* it, the first or last sample's date beyond them; null without samples.
|
||||
*/
|
||||
export function estimateDate(samples: DateSample[], article: number): Date | null {
|
||||
if (samples.length === 0) return null
|
||||
if (article <= samples[0].article) return samples[0].postedAt
|
||||
const last = samples[samples.length - 1]
|
||||
if (article >= last.article) return last.postedAt
|
||||
let lo = 0
|
||||
let hi = samples.length - 1
|
||||
while (hi - lo > 1) {
|
||||
const mid = (lo + hi) >> 1
|
||||
if (samples[mid].article <= article) lo = mid
|
||||
else hi = mid
|
||||
}
|
||||
const a = samples[lo]
|
||||
const b = samples[hi]
|
||||
const f = (article - a.article) / (b.article - a.article)
|
||||
return new Date(a.postedAt.getTime() + f * (b.postedAt.getTime() - a.postedAt.getTime()))
|
||||
}
|
||||
|
||||
/** How many samples a group gets: one per `every` articles of its span, at least `min` and at most `max`. */
|
||||
export function sampleCount(
|
||||
low: number,
|
||||
high: number,
|
||||
every: number,
|
||||
min = 16,
|
||||
max = 1000
|
||||
): number {
|
||||
return Math.min(max, Math.max(min, Math.ceil((high - low) / every) + 1))
|
||||
}
|
||||
|
||||
/**
|
||||
* Samples a group over one connection: GROUP, then an XOVER of `size` articles at each of `count` points (a slice
|
||||
* with too few articles that parse, expired or sparse, is widened to 32 × size once). The curve (monotone) and the
|
||||
* server's span.
|
||||
*/
|
||||
export async function sampleGroup(
|
||||
use: <T>(work: (client: XoverClient) => Promise<T>) => Promise<T>,
|
||||
name: string,
|
||||
options: { every: number; size: number; maxFailures?: number },
|
||||
now = new Date()
|
||||
): Promise<{
|
||||
samples: DateSample[]
|
||||
low: number
|
||||
high: number
|
||||
commands: number
|
||||
failed: string[]
|
||||
}> {
|
||||
const info = await use((client) => client.group(name))
|
||||
const samples: DateSample[] = []
|
||||
const failed: string[] = []
|
||||
let commands = 1
|
||||
// Each point on a connection of its own (GROUP again): a reply that times out (Eweka stalls some requests for
|
||||
// minutes; 16 groups never got through on one connection) costs that point, not the group.
|
||||
for (const point of samplePoints(
|
||||
info.low,
|
||||
info.high,
|
||||
sampleCount(info.low, info.high, options.every),
|
||||
options.size
|
||||
)) {
|
||||
if (failed.length >= (options.maxFailures ?? 3)) break
|
||||
try {
|
||||
const date = await use(async (client) => {
|
||||
await client.group(name)
|
||||
commands += 2
|
||||
const slice = medianDate(
|
||||
await client.xover(point, Math.min(point + options.size - 1, info.high)),
|
||||
now
|
||||
)
|
||||
if (slice) return slice
|
||||
commands++
|
||||
return medianDate(
|
||||
await client.xover(point, Math.min(point + options.size * 32 - 1, info.high)),
|
||||
now
|
||||
)
|
||||
})
|
||||
if (date) samples.push({ article: point, postedAt: date })
|
||||
} catch (error: any) {
|
||||
failed.push(`${point}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
return { samples: monotone(samples), low: info.low, high: info.high, commands, failed }
|
||||
}
|
||||
@@ -1,280 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { createReadStream, createWriteStream } from 'node:fs'
|
||||
import { mkdir, rename, stat } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import type { Readable } from 'node:stream'
|
||||
import { pipeline } from 'node:stream/promises'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import CatalogSpace from '#services/catalog_space'
|
||||
import { LeaseLost } from '#services/leases'
|
||||
import type { GroupLease, Lane, NextRange } from '#lib/edge/index'
|
||||
|
||||
export const LANES: Lane[] = ['history', 'tail']
|
||||
|
||||
/**
|
||||
* Group leases (REBUILD.md, "Voyager 3: window indexing", 1 and 5): a worker thread holds a group and asks for its
|
||||
* next range each time it finished one, so a group's ranges are indexed in article order through one carried
|
||||
* window. The coordinator keeps one row a group and lane (group_leases): the holder, its heartbeat, the range it
|
||||
* works and the snapshot its window stands at.
|
||||
*
|
||||
* - Two lanes (2026-10-27): the `history` lane works the ranges the bounds hand out (LEASE_MIN/MAX_PRIORITY,
|
||||
* LEASE_CAPTURE_ONLY: the windows, the history), the `tail` lane the tail planner's ranges (priority at or below
|
||||
* the tail's). Each lane has its own holder, window and snapshot, so a thread on the tail never wades through a
|
||||
* group's history first (a group's ranges come in article order, and a held group's history is days of ranges), and
|
||||
* the tail of a group whose history is held isn't stuck behind it. The one seam a group is where the history lane
|
||||
* reaches the tail lane's first range (its ranges are done: the history lane is caught up).
|
||||
* - lease: the free group with the most urgent planned range of the lane (the ranges' priority), and among those a
|
||||
* mix: the group with the most planned articles and the longest unheld one (never held first) take turns, so a
|
||||
* big group snapshotted at a stop doesn't wait behind hundreds of small never-held ones, and a quiet group's tail
|
||||
* is visited all the same. A holder quiet for LEASE_TTL_SECONDS is taken over. Taking a group resets what the
|
||||
* lane's snapshot doesn't cover: the lane's ranges from the snapshot's article on that the window did after it
|
||||
* (leased, done since the snapshot, or read past it) are planned again (the batch store's keys make the redo
|
||||
* idempotent); without a snapshot the window is lost and the group continues at its resume point (one seam).
|
||||
* - next: the group's lowest planned range of the lane (or the one this node already has), leased to the node; none
|
||||
* when the lane is caught up: the worker snapshots and gives the group back, and gets it again when the tail
|
||||
* planned more.
|
||||
* - snapshot: the serialized window, kept as a file on bulk (SNAPSHOT_DIR; `<group>.gz`, the tail's
|
||||
* `<group>-tail.gz`), given to the next holder of the lane.
|
||||
*/
|
||||
class GroupLeases {
|
||||
get dir(): string {
|
||||
return coordinatorConfig.snapshotDir
|
||||
}
|
||||
|
||||
/**
|
||||
* The lane's ranges, as a condition on `r`: the tail lane's are at or below the tail's priority, the history lane's
|
||||
* above it — within the lease bounds when `bounds` (what is handed out), or all of them (what a lane's window did).
|
||||
*/
|
||||
private laneFilter(lane: Lane, bounds: boolean): { sql: string; bindings: number[] } {
|
||||
const tail = coordinatorConfig.tail.priority
|
||||
if (lane === 'tail') return { sql: 'r.priority <= ?', bindings: [tail] }
|
||||
if (!bounds) return { sql: 'r.priority > ?', bindings: [tail] }
|
||||
const { leasePriorityMin: min, leasePriorityMax: max, leaseCaptureOnly } = coordinatorConfig
|
||||
return {
|
||||
sql: `r.priority > ? AND r.priority BETWEEN ? AND ?${leaseCaptureOnly ? ' AND r.capture' : ''}`,
|
||||
bindings: [tail, min, max],
|
||||
}
|
||||
}
|
||||
|
||||
async lease(nodeId: number, lane: Lane = 'history'): Promise<GroupLease | null> {
|
||||
// The catalog at its cap on the NVMe: the tail only (CatalogSpace), as the range leases did.
|
||||
if (lane === 'history' && (await CatalogSpace.full())) return null
|
||||
const rules = await RuleSets.current()
|
||||
const ttl = coordinatorConfig.leaseTtlSeconds
|
||||
const todo = this.laneFilter(lane, true)
|
||||
const all = this.laneFilter(lane, false)
|
||||
return db.transaction(async (trx) => {
|
||||
// One lease at a time: the choice reads several tables.
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
|
||||
// The free groups with planned work in the lane (a range leased by a holder gone quiet counts: a group whose
|
||||
// last range that holder took would never be taken over), by the most urgent range's priority; within a priority the
|
||||
// most planned articles and the longest unheld alternate: one lease starts from the biggest, the next from
|
||||
// the longest unheld (the sequence counts the leases; ranked afresh each time, "biggest first" would always
|
||||
// win).
|
||||
const { rows: turns } = await trx.rawQuery(`SELECT nextval('group_lease_turns') % 2 AS odd`)
|
||||
const [first, second] = Number(turns[0].odd) === 1 ? [1, 0] : [0, 1]
|
||||
const { rows } = await trx.rawQuery(
|
||||
`WITH todo AS (
|
||||
SELECT r.group_id, min(r.priority) AS priority, sum(r.end_id - r.next_id + 1) AS articles
|
||||
FROM ranges r
|
||||
WHERE (r.status = 'planned' OR (r.status = 'leased' AND r.heartbeat_at < now() - make_interval(secs => ?)))
|
||||
AND r.next_id <= r.end_id AND ${todo.sql}
|
||||
GROUP BY r.group_id
|
||||
), free AS (
|
||||
SELECT g.id, g.name, g.priority AS group_priority, l.snapshot_next, l.snapshot_at, l.heartbeat_at,
|
||||
todo.priority, todo.articles
|
||||
FROM todo JOIN groups g ON g.id = todo.group_id AND g.active
|
||||
LEFT JOIN group_leases l ON l.group_id = g.id AND l.lane = ?
|
||||
WHERE l.node_id IS NULL OR l.heartbeat_at < now() - make_interval(secs => ?)
|
||||
), turns AS (
|
||||
SELECT free.*,
|
||||
least(2 * row_number() OVER (PARTITION BY priority ORDER BY articles DESC, group_priority, id) - ?,
|
||||
2 * row_number() OVER (PARTITION BY priority ORDER BY heartbeat_at NULLS FIRST, group_priority, id) - ?) AS turn
|
||||
FROM free
|
||||
)
|
||||
SELECT * FROM turns ORDER BY priority, turn, group_priority, id LIMIT 1`,
|
||||
[ttl, ...todo.bindings, lane, ttl, first, second]
|
||||
)
|
||||
const row = rows[0]
|
||||
if (!row) return null
|
||||
const groupId = Number(row.id)
|
||||
// What the snapshot doesn't cover is read again: the lane's ranges from its article on that the window did
|
||||
// after it (a range done before the snapshot was taken — by the old range leases, say — stays done).
|
||||
if (row.snapshot_next !== null) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE ranges r SET status = 'planned', node_id = NULL, next_id = greatest(r.start_id, ?::bigint)
|
||||
WHERE r.group_id = ? AND r.end_id >= ?::bigint AND ${all.sql}
|
||||
AND (r.status = 'leased' OR (r.status = 'done' AND (r.done_at IS NULL OR r.done_at >= ?)) OR (r.status = 'planned' AND r.next_id > ?::bigint))`,
|
||||
[
|
||||
row.snapshot_next,
|
||||
groupId,
|
||||
row.snapshot_next,
|
||||
...all.bindings,
|
||||
row.snapshot_at ?? new Date(0),
|
||||
row.snapshot_next,
|
||||
]
|
||||
)
|
||||
} else {
|
||||
await trx.rawQuery(
|
||||
`UPDATE ranges r SET status = 'planned', node_id = NULL WHERE r.group_id = ? AND r.status = 'leased' AND ${all.sql}`,
|
||||
[groupId, ...all.bindings]
|
||||
)
|
||||
}
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO group_leases (group_id, lane, node_id, leased_at, heartbeat_at, range_id)
|
||||
VALUES (?, ?, ?, now(), now(), NULL)
|
||||
ON CONFLICT (group_id, lane) DO UPDATE SET node_id = excluded.node_id, leased_at = now(), heartbeat_at = now(), range_id = NULL`,
|
||||
[groupId, lane, nodeId]
|
||||
)
|
||||
await trx.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
return {
|
||||
group: { id: groupId, name: row.name, priority: Number(row.group_priority) },
|
||||
lane,
|
||||
rules,
|
||||
snapshot: row.snapshot_next !== null ? { next: Number(row.snapshot_next) } : null,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** The group's lease row of the lane, when this node holds it. */
|
||||
async held(groupId: number, nodeId: number, lane: Lane = 'history') {
|
||||
const lease = await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.first()
|
||||
if (!lease || Number(lease.node_id) !== nodeId)
|
||||
throw new LeaseLost(`Group ${groupId} (${lane}) is not held by this node`)
|
||||
return lease
|
||||
}
|
||||
|
||||
/**
|
||||
* The group's next range of the lane in article order (or the one this node has), leased to the node; null when
|
||||
* the lane is caught up.
|
||||
*/
|
||||
async next(groupId: number, nodeId: number, lane: Lane = 'history'): Promise<NextRange | null> {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
const rules = await RuleSets.current()
|
||||
const todo = this.laneFilter(lane, true)
|
||||
const all = this.laneFilter(lane, false)
|
||||
const mine = (
|
||||
await db.rawQuery(
|
||||
`SELECT r.id, r.start_id, r.end_id, r.next_id FROM ranges r
|
||||
WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql} ORDER BY r.start_id LIMIT 1`,
|
||||
[groupId, nodeId, ...all.bindings]
|
||||
)
|
||||
).rows[0]
|
||||
const row =
|
||||
mine ??
|
||||
(
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
|
||||
WHERE r.id = (
|
||||
SELECT r.id FROM ranges r WHERE r.group_id = ? AND r.status = 'planned' AND r.next_id <= r.end_id AND ${todo.sql}
|
||||
ORDER BY r.start_id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
RETURNING r.id, r.start_id, r.end_id, r.next_id`,
|
||||
[nodeId, rules.version, groupId, ...todo.bindings]
|
||||
)
|
||||
).rows[0]
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ range_id: row ? Number(row.id) : null, heartbeat_at: new Date() })
|
||||
if (!row) return null
|
||||
return {
|
||||
range: {
|
||||
id: Number(row.id),
|
||||
start: Number(row.start_id),
|
||||
end: Number(row.end_id),
|
||||
next: Number(row.next_id),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async heartbeat(groupId: number, nodeId: number, lane: Lane = 'history') {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ heartbeat_at: new Date() })
|
||||
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
}
|
||||
|
||||
/** The lane a range belongs to, by its priority. */
|
||||
laneOf(priority: number): Lane {
|
||||
return priority <= coordinatorConfig.tail.priority ? 'tail' : 'history'
|
||||
}
|
||||
|
||||
/** A range of the group finished by its holder: counted on the lane's lease (Leases.finish does the range). */
|
||||
async rangeDone(groupId: number, headers: number, lane: Lane = 'history') {
|
||||
await db.rawQuery(
|
||||
`UPDATE group_leases SET ranges_done = ranges_done + 1, headers = headers + ?, range_id = NULL WHERE group_id = ? AND lane = ?`,
|
||||
[headers, groupId, lane]
|
||||
)
|
||||
}
|
||||
|
||||
/** The worker gives the group back (after its snapshot): its range stays at its resume point for the next holder. */
|
||||
async release(groupId: number, nodeId: number, lane: Lane = 'history') {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
const all = this.laneFilter(lane, false)
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges r SET status = 'planned', node_id = NULL
|
||||
WHERE r.group_id = ? AND r.status = 'leased' AND r.node_id = ? AND ${all.sql}`,
|
||||
[groupId, nodeId, ...all.bindings]
|
||||
)
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ node_id: null, range_id: null })
|
||||
}
|
||||
|
||||
snapshotPath(groupId: number, lane: Lane = 'history') {
|
||||
return join(this.dir, lane === 'tail' ? `${groupId}-tail.gz` : `${groupId}.gz`)
|
||||
}
|
||||
|
||||
/**
|
||||
* The window's snapshot standing at `next` (the next article to index), streamed to disk (a busy group's is
|
||||
* gigabytes) and written whole before it replaces the old one.
|
||||
*/
|
||||
async putSnapshot(
|
||||
groupId: number,
|
||||
nodeId: number,
|
||||
next: number,
|
||||
body: Readable,
|
||||
lane: Lane = 'history'
|
||||
) {
|
||||
await this.held(groupId, nodeId, lane)
|
||||
await mkdir(this.dir, { recursive: true })
|
||||
const path = this.snapshotPath(groupId, lane)
|
||||
await pipeline(body, createWriteStream(`${path}.part`))
|
||||
await rename(`${path}.part`, path)
|
||||
await db
|
||||
.from('group_leases')
|
||||
.where('group_id', groupId)
|
||||
.where('lane', lane)
|
||||
.update({ snapshot_next: next, snapshot_at: new Date(), heartbeat_at: new Date() })
|
||||
}
|
||||
|
||||
/** The snapshot's stream and size, or null when there is none. */
|
||||
async getSnapshot(
|
||||
groupId: number,
|
||||
nodeId: number,
|
||||
lane: Lane = 'history'
|
||||
): Promise<{ stream: Readable; size: number; next: number } | null> {
|
||||
const lease = await this.held(groupId, nodeId, lane)
|
||||
if (lease.snapshot_next === null) return null
|
||||
const path = this.snapshotPath(groupId, lane)
|
||||
try {
|
||||
const { size } = await stat(path)
|
||||
return { stream: createReadStream(path), size, next: Number(lease.snapshot_next) }
|
||||
} catch {
|
||||
return null
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new GroupLeases()
|
||||
@@ -1,27 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/**
|
||||
* Twins by content (LOG.md (130)): a probed file's first 16 KiB and size (file_hashes, what the par2 merge and the
|
||||
* repeated verdict match) in a catalog release that has a name, readable or found inside. The same film posted
|
||||
* under scrambles by other tools gets that name without another fetch. Releases with a junk verdict don't count.
|
||||
*/
|
||||
export async function twinNames(
|
||||
pairs: { hash16k: string; size: number }[],
|
||||
trx: { rawQuery: typeof db.rawQuery } = db
|
||||
): Promise<Map<string, string>> {
|
||||
const out = new Map<string, string>()
|
||||
if (pairs.length === 0) return out
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT DISTINCT ON (h.hash16k, h.size) h.hash16k, h.size, coalesce(r.inner_name, r.name) AS name
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(hash16k text, size bigint)
|
||||
JOIN file_hashes h ON h.hash16k = v.hash16k AND h.size = v.size
|
||||
JOIN releases r ON r.id = h.release_id AND (r.readable OR r.inner_name IS NOT NULL)
|
||||
WHERE NOT EXISTS (SELECT 1 FROM release_verdicts x WHERE x.release_id = r.id)
|
||||
ORDER BY h.hash16k, h.size, r.readable DESC, r.id`,
|
||||
[JSON.stringify(pairs)]
|
||||
)
|
||||
for (const r of rows) out.set(`${r.hash16k}\t${r.size}`, r.name)
|
||||
return out
|
||||
}
|
||||
|
||||
export const twinKey = (hash16k: string, size: number) => `${hash16k}\t${size}`
|
||||
@@ -1,163 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import Spool from '#services/spool'
|
||||
import Corpus from '#services/corpus'
|
||||
import CatalogSpace from '#services/catalog_space'
|
||||
import type { Lease } from '#lib/edge/index'
|
||||
|
||||
export class LeaseLost extends Error {}
|
||||
|
||||
/**
|
||||
* Ranges for workers (REBUILD.md, coordinator): the next range with chunks
|
||||
* spooled, by priority (oldest era first), else the next planned one (the
|
||||
* prefetcher fetches leased ranges first; the worker's stream waits). A
|
||||
* worker never waits for prefetching to choose its range: refused leases
|
||||
* left workers idle for 30 s, so fewer ranges were leased and fewer
|
||||
* prefetched (LOG.md 2026-10-09). A lease whose worker went quiet for
|
||||
* LEASE_TTL_SECONDS is taken over and resumes at its resume point.
|
||||
*/
|
||||
/**
|
||||
* A group's batches wait for each other (the batch store locks the group): more workers than this on one group only
|
||||
* queue. Eight on one picture group's gap ranges (priority -1, in id order) held the node API's batches for minutes
|
||||
* (LOG.md 2026-10-16 (68)).
|
||||
*/
|
||||
const MAX_LEASES_PER_GROUP = 2
|
||||
|
||||
class Leases {
|
||||
async lease(nodeId: number): Promise<Lease | null> {
|
||||
const rules = await RuleSets.current()
|
||||
// The catalog at its cap on the NVMe: the tail and the stored ranges only (CatalogSpace).
|
||||
const history = (await CatalogSpace.full()) ? 'AND priority < 0' : ''
|
||||
const take = (spooled: boolean) =>
|
||||
db.rawQuery(
|
||||
`UPDATE ranges r SET status = 'leased', node_id = ?, leased_at = now(), heartbeat_at = now(), attempts = attempts + 1, rule_version = ?
|
||||
WHERE r.id = (
|
||||
SELECT id FROM ranges
|
||||
WHERE ${spooled ? 'spooled_chunks > 0' : 'true'} AND next_id <= end_id ${history}
|
||||
AND (status = 'planned' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))
|
||||
AND (SELECT count(*) FROM ranges l
|
||||
WHERE l.status = 'leased' AND l.group_id = ranges.group_id AND l.id <> ranges.id
|
||||
AND l.heartbeat_at >= now() - make_interval(secs => ?)) < ${MAX_LEASES_PER_GROUP}
|
||||
ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||
RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
|
||||
[
|
||||
nodeId,
|
||||
rules.version,
|
||||
coordinatorConfig.leaseTtlSeconds,
|
||||
coordinatorConfig.leaseTtlSeconds,
|
||||
]
|
||||
)
|
||||
let { rows } = await take(true)
|
||||
if (rows.length === 0) ({ rows } = await take(false))
|
||||
const row = rows[0]
|
||||
if (!row) return null
|
||||
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
const group = await db.from('groups').where('id', row.group_id).select('name').firstOrFail()
|
||||
return {
|
||||
range: {
|
||||
id: Number(row.id),
|
||||
group: group.name,
|
||||
start: Number(row.start_id),
|
||||
end: Number(row.end_id),
|
||||
next: Number(row.next_id),
|
||||
},
|
||||
rules,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ranges read to their end whose worker went quiet before finishing them: done. The lease query skips a range past
|
||||
* its end, so one whose finish never came stayed leased for good (a gap range since 2026-10-16 14:03, during the lock
|
||||
* pile-up; LOG.md 2026-10-17 (72)). Its batches are all stored; its spooled chunks go.
|
||||
*/
|
||||
async finishAbandoned(): Promise<number> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE ranges SET status = 'done', done_at = now()
|
||||
WHERE status = 'leased' AND next_id > end_id AND heartbeat_at < now() - make_interval(secs => ?)
|
||||
RETURNING id`,
|
||||
[coordinatorConfig.leaseTtlSeconds]
|
||||
)
|
||||
for (const { id } of rows) await Spool.remove(Number(id))
|
||||
return rows.length
|
||||
}
|
||||
|
||||
async leased(rangeId: number, nodeId: number) {
|
||||
const range = await db.from('ranges').where('id', rangeId).first()
|
||||
if (!range || range.status !== 'leased' || Number(range.node_id) !== nodeId)
|
||||
throw new LeaseLost(`Range ${rangeId} is not leased to this node`)
|
||||
return range
|
||||
}
|
||||
|
||||
async heartbeat(rangeId: number, nodeId: number) {
|
||||
await this.leased(rangeId, nodeId)
|
||||
await db.from('ranges').where('id', rangeId).update({ heartbeat_at: new Date() })
|
||||
await db.from('nodes').where('id', nodeId).update({ last_seen_at: new Date() })
|
||||
}
|
||||
|
||||
/** The worker reached the range's end: done, and its spooled chunks go. */
|
||||
async finish(rangeId: number, nodeId: number) {
|
||||
const range = await this.leased(rangeId, nodeId)
|
||||
if (Number(range.next_id) <= Number(range.end_id))
|
||||
throw new LeaseLost(
|
||||
`Range ${rangeId} is at ${range.next_id}, not past its end ${range.end_id}`
|
||||
)
|
||||
await db
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
.update({ status: 'done', done_at: new Date(), heartbeat_at: new Date() })
|
||||
await Spool.remove(rangeId)
|
||||
}
|
||||
|
||||
/**
|
||||
* A worker got a chunk that doesn't inflate: the spooled chunk goes and
|
||||
* the range back to the queue (the prefetcher fetches leased ranges'
|
||||
* missing chunks again; the next worker resumes at the range's resume point).
|
||||
*/
|
||||
async corrupt(rangeId: number, nodeId: number, from: number) {
|
||||
const range = await this.leased(rangeId, nodeId)
|
||||
const span = Spool.chunksOf({ start: Number(range.start_id), end: Number(range.end_id) }).find(
|
||||
(s) => s.from === from
|
||||
)
|
||||
if (span) {
|
||||
await Spool.removeChunk(rangeId, span)
|
||||
if (range.capture) {
|
||||
const group = await db.from('groups').where('id', range.group_id).select('name').first()
|
||||
await Corpus.removeChunk(
|
||||
{ group: group.name, start: Number(range.start_id), end: Number(range.end_id) },
|
||||
span
|
||||
)
|
||||
// Not complete in the corpus any more: the corpus fill fetches the chunk again.
|
||||
await db.from('ranges').where('id', rangeId).update({ stored_at: null })
|
||||
}
|
||||
}
|
||||
const plain: number[] = (range.plain_chunks ?? []).map(Number)
|
||||
// Reported once already: the compressed copy may have been in flight (or the prefetcher's targets not
|
||||
// refreshed yet) and spooled again (range 161820, LOG.md 2026-10-11), so a second try plain first.
|
||||
if (plain.filter((p) => p === from).length >= 2) {
|
||||
// Fetched plain (deflated here) and still not inflating: not the server's compression; set aside.
|
||||
await db
|
||||
.from('ranges')
|
||||
.where('id', rangeId)
|
||||
.update({
|
||||
status: 'failed',
|
||||
node_id: null,
|
||||
error: `chunk ${from} doesn't inflate, also fetched plain`,
|
||||
})
|
||||
return
|
||||
}
|
||||
// The server's compressed reply is cut short (the same bytes every time): fetched plain next time.
|
||||
await db.rawQuery(
|
||||
`UPDATE ranges SET status = 'planned', node_id = NULL, plain_chunks = array_append(plain_chunks, ?::bigint) WHERE id = ?`,
|
||||
[from, rangeId]
|
||||
)
|
||||
}
|
||||
|
||||
/** A stopping worker gives its range back; the next one resumes it. */
|
||||
async release(rangeId: number, nodeId: number) {
|
||||
await this.leased(rangeId, nodeId)
|
||||
await db.from('ranges').where('id', rangeId).update({ status: 'planned', node_id: null })
|
||||
}
|
||||
}
|
||||
|
||||
export default new Leases()
|
||||
@@ -1,230 +0,0 @@
|
||||
import env from '#start/env'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import type { NntpShare } from '#services/nntp_shares'
|
||||
|
||||
/** Connections wanted beyond the most in use at once since the last sync (the planner's, a hedge's). */
|
||||
const SHARE_MARGIN = 2
|
||||
|
||||
/**
|
||||
* The coordinator's header connections (REBUILD.md, decision e: workers
|
||||
* have no NNTP account). At most NNTP_CONNECTIONS, reused; a connection
|
||||
* that failed, or is older than NNTP_CONNECTION_MAX_MINUTES, is closed and
|
||||
* replaced on the next use. Sharing the account (share()), at most its
|
||||
* share: over it, connections close as they come free. One left free for
|
||||
* NNTP_IDLE_CLOSE_SECONDS is closed on the next sync.
|
||||
*/
|
||||
class NntpPool {
|
||||
private idle: XoverClient[] = []
|
||||
private busy = new Set<XoverClient>()
|
||||
private open = 0
|
||||
private waiting: ((client: XoverClient) => void)[] = []
|
||||
private openedAt = new Map<XoverClient, number>()
|
||||
/** When a connection gets closed once free: its max age ±25%, so they don't all reconnect at once. */
|
||||
private retireAt = new Map<XoverClient, number>()
|
||||
/** When an idle connection came free. */
|
||||
private idleSince = new Map<XoverClient, number>()
|
||||
private limit = coordinatorConfig.nntpConnections
|
||||
/** Since the last sync: the most connections in use at once, and whether work waited for one (starved()). */
|
||||
private peak = 0
|
||||
private short = false
|
||||
|
||||
get size() {
|
||||
return this.limit
|
||||
}
|
||||
|
||||
/** Work that could use another connection didn't get one (the prefetcher with all its loops running). */
|
||||
starved() {
|
||||
this.short = true
|
||||
}
|
||||
|
||||
/**
|
||||
* Tells the share what the pool could use and has open, and takes the share's grant as its size. After work waited:
|
||||
* twice its size (history going on gets its maximum in a few syncs; a tail round, done in seconds, doesn't take
|
||||
* the prober's connections for one); else the most in use at once since the last sync and a margin.
|
||||
*/
|
||||
async share(share: NntpShare) {
|
||||
const want = this.short || this.waiting.length > 0 ? Math.max(this.limit * 2, this.limit + SHARE_MARGIN) : this.peak + SHARE_MARGIN
|
||||
this.short = false
|
||||
this.peak = this.busy.size
|
||||
const granted = await share.sync(want, this.open)
|
||||
const was = this.limit
|
||||
this.limit = granted
|
||||
// Over the share: the idle ones close now, the busy ones once free (release()).
|
||||
while (this.open > this.limit && this.idle.length > 0) this.retire(this.idle.pop()!)
|
||||
this.closeIdle()
|
||||
// Under it with work waiting: connections for it.
|
||||
while (this.waiting.length > 0 && this.open < this.limit) {
|
||||
const next = this.waiting.shift()!
|
||||
this.acquire().then(next, () => this.waiting.unshift(next))
|
||||
}
|
||||
return { was, granted }
|
||||
}
|
||||
|
||||
async use<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
return this.run(await this.acquire(), work)
|
||||
}
|
||||
|
||||
/** As use(), if a connection is free or can be opened now; else null, without waiting. */
|
||||
tryUse<T>(work: (client: XoverClient) => Promise<T>): Promise<T> | null {
|
||||
if (this.idle.length === 0 && this.open >= this.size) return null
|
||||
return this.acquire().then((client) => this.run(client, work))
|
||||
}
|
||||
|
||||
/** Connections with a command in flight (the prefetcher's, the planner's). */
|
||||
get busyCount() {
|
||||
return this.busy.size
|
||||
}
|
||||
|
||||
/** How long ago the connection was opened, in ms. */
|
||||
ageOf(client: XoverClient): number {
|
||||
return Date.now() - (this.openedAt.get(client) ?? Date.now())
|
||||
}
|
||||
|
||||
private async run<T>(client: XoverClient, work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
this.busy.add(client)
|
||||
this.peak = Math.max(this.peak, this.busy.size)
|
||||
let broken = false
|
||||
try {
|
||||
return await work(client)
|
||||
} catch (error) {
|
||||
broken = true
|
||||
throw error
|
||||
} finally {
|
||||
this.busy.delete(client)
|
||||
this.release(client, broken)
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes the connections free for nntpIdleCloseMs, the plain one too. */
|
||||
private closeIdle() {
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (idleMs <= 0) return
|
||||
const before = Date.now() - idleMs
|
||||
for (const client of this.idle.filter((c) => (this.idleSince.get(c) ?? Infinity) <= before)) {
|
||||
this.idle.splice(this.idle.indexOf(client), 1)
|
||||
this.retire(client)
|
||||
}
|
||||
if (this.plain && this.plainPending === 0 && this.plainUsedAt <= before) {
|
||||
void this.plain.then((c) => c.close()).catch(() => {})
|
||||
this.plain = null
|
||||
}
|
||||
}
|
||||
|
||||
private async acquire(): Promise<XoverClient> {
|
||||
// The one used last: the others stay free long enough to close.
|
||||
const ready = this.idle.pop()
|
||||
if (ready) {
|
||||
this.idleSince.delete(ready)
|
||||
return ready
|
||||
}
|
||||
if (this.open < this.size) {
|
||||
this.open++
|
||||
try {
|
||||
const client = await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: true,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
this.openedAt.set(client, Date.now())
|
||||
const maxMs = coordinatorConfig.nntpConnectionMaxMs
|
||||
if (maxMs > 0) this.retireAt.set(client, Date.now() + maxMs * (0.75 + Math.random() / 2))
|
||||
return client
|
||||
} catch (error) {
|
||||
this.open--
|
||||
throw error
|
||||
}
|
||||
}
|
||||
return new Promise((resolve) => this.waiting.push(resolve))
|
||||
}
|
||||
|
||||
private release(client: XoverClient, broken: boolean) {
|
||||
if (this.open > this.limit) {
|
||||
this.retire(client)
|
||||
return
|
||||
}
|
||||
if (broken || Date.now() >= (this.retireAt.get(client) ?? Infinity)) {
|
||||
this.retire(client)
|
||||
// Someone waiting gets a fresh connection.
|
||||
const next = this.waiting.shift()
|
||||
if (next) this.acquire().then(next, () => this.waiting.unshift(next))
|
||||
return
|
||||
}
|
||||
const next = this.waiting.shift()
|
||||
if (next) next(client)
|
||||
else {
|
||||
this.idle.push(client)
|
||||
this.idleSince.set(client, Date.now())
|
||||
}
|
||||
}
|
||||
|
||||
private retire(client: XoverClient) {
|
||||
this.open--
|
||||
this.openedAt.delete(client)
|
||||
this.retireAt.delete(client)
|
||||
this.idleSince.delete(client)
|
||||
try {
|
||||
client.close()
|
||||
} catch (error: any) {
|
||||
logger.debug(`Closing an NNTP connection: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Closes every connection, also those in use: their commands fail. */
|
||||
private plain: Promise<XoverClient> | null = null
|
||||
private plainQueue: Promise<unknown> = Promise.resolve()
|
||||
/** Commands queued or running on the plain connection, and when the last one ended. */
|
||||
private plainPending = 0
|
||||
private plainUsedAt = 0
|
||||
|
||||
/**
|
||||
* One connection without XFEATURE COMPRESS GZIP, opened when first needed,
|
||||
* one command at a time: for chunks whose compressed reply the server
|
||||
* cuts short (Leases.corrupt). Its replies are deflated here.
|
||||
*/
|
||||
async usePlain<T>(work: (client: XoverClient) => Promise<T>): Promise<T> {
|
||||
this.plainPending++
|
||||
const run = this.plainQueue.then(async () => {
|
||||
this.plain ??= XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
const client = await this.plain
|
||||
try {
|
||||
return await work(client)
|
||||
} catch (error) {
|
||||
this.plain = null
|
||||
client.close()
|
||||
throw error
|
||||
}
|
||||
})
|
||||
this.plainQueue = run.catch(() => {}).finally(() => {
|
||||
this.plainPending--
|
||||
this.plainUsedAt = Date.now()
|
||||
})
|
||||
return run
|
||||
}
|
||||
|
||||
async close() {
|
||||
void this.plain?.then((c) => c.close()).catch(() => {})
|
||||
this.plain = null
|
||||
for (const client of [...this.idle.splice(0), ...this.busy]) client.close()
|
||||
this.busy.clear()
|
||||
this.openedAt.clear()
|
||||
this.retireAt.clear()
|
||||
this.idleSince.clear()
|
||||
this.open = 0
|
||||
}
|
||||
}
|
||||
|
||||
export default new NntpPool()
|
||||
@@ -1,90 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
/** A share whose process hasn't synced for this long is gone: its connections closed with it. */
|
||||
const STALE_SECONDS = 60
|
||||
/** Syncs one at a time (pg_advisory_xact_lock). */
|
||||
const LOCK = 2_026_101_875
|
||||
|
||||
export interface Share {
|
||||
name: string
|
||||
priority: number
|
||||
/** Kept whatever the others want (as far as the total goes, by priority). */
|
||||
min: number
|
||||
/** What it could use now. */
|
||||
want: number
|
||||
/** Connections it has open (or may have: a lane without one yet counts). */
|
||||
open: number
|
||||
}
|
||||
|
||||
/** What each share should have: its minimum first, then by priority what it wants, up to the total. */
|
||||
export function targets(total: number, shares: Share[]): Map<string, number> {
|
||||
const ordered = [...shares].sort((a, b) => b.priority - a.priority || a.name.localeCompare(b.name))
|
||||
const target = new Map<string, number>()
|
||||
let left = total
|
||||
for (const s of ordered) {
|
||||
const t = Math.min(s.min, left)
|
||||
target.set(s.name, t)
|
||||
left -= t
|
||||
}
|
||||
for (const s of ordered) {
|
||||
const more = Math.min(Math.max(s.want - target.get(s.name)!, 0), left)
|
||||
target.set(s.name, target.get(s.name)! + more)
|
||||
left -= more
|
||||
}
|
||||
return target
|
||||
}
|
||||
|
||||
/**
|
||||
* The connections `self` may have: its target, but no more than the others leave over. Another share holds what it
|
||||
* has open or was granted, whichever is more (it may open up to its grant until its next sync tells it otherwise),
|
||||
* so the account never goes over the total: a share grows only into what the others closed or gave up.
|
||||
*/
|
||||
export function grant(total: number, self: Share, others: (Share & { granted: number })[]): number {
|
||||
const target = targets(total, [self, ...others]).get(self.name)!
|
||||
const held = others.reduce((sum, o) => sum + Math.max(o.open, o.granted), 0)
|
||||
return Math.max(0, Math.min(target, total - held))
|
||||
}
|
||||
|
||||
/**
|
||||
* A process's share of the NNTP account's connections (LOG.md 2026-10-18 (75)): the coordinator and the prober had a
|
||||
* fixed split, and with history paused the coordinator's 38 stood idle while the prober's 8 were its limit. Each
|
||||
* syncs every nntpShares.everyMs (what it wants and has open) and gets what it may open; one wanting less closes
|
||||
* connections as they come free, and the other grows as those show up closed.
|
||||
*/
|
||||
export class NntpShare {
|
||||
granted = 0
|
||||
|
||||
constructor(
|
||||
readonly name: string,
|
||||
private priority: number,
|
||||
readonly min: number,
|
||||
readonly max: number
|
||||
) {}
|
||||
|
||||
async sync(want: number, open: number): Promise<number> {
|
||||
const self: Share = { name: this.name, priority: this.priority, min: this.min, want: Math.min(Math.max(want, this.min), this.max), open }
|
||||
const total = coordinatorConfig.nntpShares.total
|
||||
this.granted = await db.transaction(async (trx) => {
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(?)`, [LOCK])
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT name, priority, min, want, open, granted FROM nntp_shares WHERE name <> ? AND heartbeat_at > now() - make_interval(secs => ?)`,
|
||||
[this.name, STALE_SECONDS]
|
||||
)
|
||||
const granted = grant(total, self, rows)
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO nntp_shares (name, priority, min, want, open, granted, heartbeat_at) VALUES (?, ?, ?, ?, ?, ?, now())
|
||||
ON CONFLICT (name) DO UPDATE SET priority = excluded.priority, min = excluded.min, want = excluded.want,
|
||||
open = excluded.open, granted = excluded.granted, heartbeat_at = now()`,
|
||||
[this.name, this.priority, this.min, self.want, open, granted]
|
||||
)
|
||||
return granted
|
||||
})
|
||||
return this.granted
|
||||
}
|
||||
|
||||
/** Stopping: its connections are closed, the others needn't wait for the row to go stale. */
|
||||
async leave() {
|
||||
await db.from('nntp_shares').where('name', this.name).delete()
|
||||
}
|
||||
}
|
||||
@@ -1,65 +0,0 @@
|
||||
import { gunzipSync } from 'node:zlib'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
|
||||
import { buildNzb } from '#lib/nzb/index'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
|
||||
/** Where a known password came from, the most trusted first: the spot's NZB meta, its title, its text. */
|
||||
const PASSWORD_SOURCES = ['nzb', 'spot-title', 'spot-text']
|
||||
|
||||
/**
|
||||
* A release's files and NZB (REBUILD.md, part 9), from its chunks in the
|
||||
* chunk store: every chunk decoded, its files in name order. A file stored
|
||||
* twice (the same name and first article, e.g. a range read again) is taken
|
||||
* once. A known password (release_passwords) goes into the NZB's meta, where
|
||||
* SABnzbd and NZBGet take it for unpacking.
|
||||
*/
|
||||
class NzbService {
|
||||
/** The release's known passwords, the most trusted first. */
|
||||
async passwords(releaseId: number): Promise<{ password: string; source: string }[]> {
|
||||
const rows = await db.from('release_passwords').where('release_id', releaseId).orderBy('created_at').select('password', 'source')
|
||||
const rank = (source: string) => (PASSWORD_SOURCES.indexOf(source) + PASSWORD_SOURCES.length + 1) % (PASSWORD_SOURCES.length + 1)
|
||||
return rows.sort((a, b) => rank(a.source) - rank(b.source)).map((r) => ({ password: r.password, source: r.source }))
|
||||
}
|
||||
|
||||
async files(releaseId: number, client: any = db): Promise<ChunkFile[]> {
|
||||
const chunks = await client.from('release_chunks').where('release_id', releaseId).orderBy('id').select('payload')
|
||||
const files = new Map<string, ChunkFile>()
|
||||
for (const chunk of chunks) {
|
||||
for (const file of decodeChunk(chunk.payload)) files.set(`${file.name}\n${file.segments[0]?.[1] ?? ''}`, file)
|
||||
}
|
||||
return [...files.values()].sort((a, b) => a.name.localeCompare(b.name))
|
||||
}
|
||||
|
||||
async build(releaseId: number): Promise<{ name: string; xml: string } | null> {
|
||||
const release = await db.from('releases').where('id', releaseId).select('name', 'source').first()
|
||||
if (!release) return null
|
||||
const password = (await this.passwords(releaseId))[0]?.password
|
||||
// A release a spot made of its own (no files matched the header pass's): the spot's NZB as read.
|
||||
if (Number(release.source) === SOURCES.spot) {
|
||||
const spot = await db.from('spots').where('release_id', releaseId).whereNotNull('nzb').select('nzb').first()
|
||||
if (spot) return { name: release.name, xml: withPassword(gunzipSync(spot.nzb).toString('utf8'), password) }
|
||||
}
|
||||
const xml = buildNzb(
|
||||
(await this.files(releaseId)).map((f) => ({
|
||||
name: f.name,
|
||||
poster: f.poster,
|
||||
date: f.postedAt ?? 0,
|
||||
groups: f.groups,
|
||||
segments: f.segments.map(([number, id, size]) => ({ number, id, size })),
|
||||
})),
|
||||
{ title: release.name, password }
|
||||
)
|
||||
return { name: release.name, xml }
|
||||
}
|
||||
}
|
||||
|
||||
/** A read NZB with the password in its meta, when it has none (the spot's title or text named it, not its NZB). */
|
||||
export function withPassword(xml: string, password: string | undefined): string {
|
||||
if (!password || /<meta\s+type=["']password["']/i.test(xml)) return xml
|
||||
const meta = `<meta type="password">${password.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')}</meta>`
|
||||
if (/<head\b[^>]*>/i.test(xml)) return xml.replace(/<head\b[^>]*>/i, (head) => `${head}${meta}`)
|
||||
return xml.replace(/<nzb\b[^>]*>/i, (nzb) => `${nzb}<head>${meta}</head>`)
|
||||
}
|
||||
|
||||
export default new NzbService()
|
||||
@@ -1,53 +0,0 @@
|
||||
import NzbService from '#services/nzb_service'
|
||||
|
||||
/** A file a par2 set describes (par2_files). */
|
||||
export interface SetFile {
|
||||
name: string
|
||||
length: number
|
||||
hash16k: string
|
||||
}
|
||||
|
||||
const base = (name: string) => name.replace(/^.*[\\/]/, '').toLowerCase()
|
||||
|
||||
/**
|
||||
* Whether a release holds every file its par2 sets describe (REBUILD.md, part 8): for one without an [i/N] counter,
|
||||
* whose completeness the settler leaves unknown. A described file is there when a file of the release has its name,
|
||||
* or a probed file of it its first 16 KiB and size (file_hashes). An upload renamed after its par2 was made (no
|
||||
* described name among its files) is complete when it has as many files besides par2 as the set describes, and as
|
||||
* many bytes (its articles' bytes, a little over the files' with yEnc).
|
||||
*/
|
||||
export function par2Complete(set: SetFile[], files: { name: string; size: number }[], hashes: { hash16k: string; size: number }[]): boolean {
|
||||
const names = new Set(files.map((f) => base(f.name)))
|
||||
const hashed = new Set(hashes.map((h) => `${h.hash16k}:${h.size}`))
|
||||
const byName = set.filter((f) => names.has(base(f.name))).length
|
||||
if (set.every((f) => names.has(base(f.name)) || hashed.has(`${f.hash16k}:${f.length}`))) return true
|
||||
if (byName > 0) return false
|
||||
const data = files.filter((f) => !/\.par2$/i.test(f.name))
|
||||
return data.length >= set.length && data.reduce((n, f) => n + f.size, 0) >= set.reduce((n, f) => n + f.length, 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides `complete` for the settled releases among `releaseIds` that have no counter and a par2 set with its file
|
||||
* list (par2_files; the pruner drops it once the set is done, so it decides before). How many it decided. `client`:
|
||||
* a transaction that moved chunks (a merge) reads them as moved.
|
||||
*/
|
||||
export async function decidePar2Completeness(client: any, releaseIds: (number | string)[]): Promise<number> {
|
||||
if (releaseIds.length === 0) return 0
|
||||
const { rows: releases } = await client.rawQuery(
|
||||
`SELECT id FROM releases WHERE id = ANY(?) AND file_total IS NULL AND settled_at IS NOT NULL`,
|
||||
[releaseIds.map(String)]
|
||||
)
|
||||
let decided = 0
|
||||
for (const { id } of releases) {
|
||||
const { rows: set } = await client.rawQuery(
|
||||
`SELECT DISTINCT f.file_id, f.name, f.length::float8 AS length, f.hash16k FROM par2_sets s JOIN par2_files f ON f.set_id = s.set_id WHERE s.release_id = ?`,
|
||||
[id]
|
||||
)
|
||||
if (set.length === 0) continue
|
||||
const { rows: hashes } = await client.rawQuery(`SELECT hash16k, size::float8 AS size FROM file_hashes WHERE release_id = ?`, [id])
|
||||
const files = (await NzbService.files(Number(id), client)).map((f) => ({ name: f.name, size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0) }))
|
||||
await client.rawQuery(`UPDATE releases SET complete = ? WHERE id = ?`, [par2Complete(set, files, hashes), id])
|
||||
decided++
|
||||
}
|
||||
return decided
|
||||
}
|
||||
@@ -1,498 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { par2ReleaseName } from '#lib/release/index'
|
||||
import { looksRandom } from '#lib/release/random'
|
||||
import { SidecarFolder, SIDECAR } from '#services/sidecar_folder'
|
||||
import { randomPoster } from '#lib/release/identity'
|
||||
import NzbService from '#services/nzb_service'
|
||||
import { Par2Pruner } from '#services/par2_pruner'
|
||||
import { pruneSevenZipTails } from '#services/seven_zip_tails'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
|
||||
const MAX_SPREAD_MS = 48 * 3600 * 1000
|
||||
/**
|
||||
* A set held by more releases nearby than this is one posted again and again, not one upload's par2 (every
|
||||
* "kantodoNNN.bin" test upload carries the same set: 64 releases became one of 631 files; LOG.md 2026-10-12 (30)).
|
||||
*/
|
||||
const MAX_HOLDERS = 4
|
||||
/** Members with more files than this many times the set's, plus SLACK_FILES (its par2 volumes), are more than one upload. */
|
||||
const MAX_FILES_PER_SET_FILE = 3
|
||||
const SLACK_FILES = 10
|
||||
/**
|
||||
* A par2 file held by more file rows than this (the same first 16 KiB and size) is a common file, not one upload's:
|
||||
* it matches nothing. One such file was in 190,380 uploads, and a set naming it loaded every one of them; the merger
|
||||
* ran out of heap every few minutes (LOG.md 2026-10-20 (85)). The judge calls 20 releases repeated.
|
||||
*/
|
||||
const COMMON_FILE = 20
|
||||
/**
|
||||
* The name pass: a set's own poster's releases in its group posted this close to it are looked at; at most
|
||||
* NAME_CANDIDATES of them (a poster's uploads of a day).
|
||||
*/
|
||||
const NAME_SPREAD_MS = 12 * 3600 * 1000
|
||||
const NAME_CANDIDATES = 200
|
||||
/** Files a set's upload carries besides those it describes: sidecars, its par2 files, a cover or preview. */
|
||||
const besideSet = (name: string) => SIDECAR.test(name) || /\.(par2|jpe?g|png)$/i.test(name)
|
||||
/** Between rounds. */
|
||||
const ROUND_MS = 60_000
|
||||
/** A window looks back this far past the previous one's end (rows committed after it was read). */
|
||||
const OVERLAP_MS = 60_000
|
||||
/**
|
||||
* A round goes through what's new in windows of this much (created_at), its place kept after each: a round behind by
|
||||
* two days of the probe backlog (3.5M sets) never ended, and every restart started it over (LOG.md 2026-10-20 (85)).
|
||||
*/
|
||||
const WINDOW_MS = 60 * 60_000
|
||||
/** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
|
||||
const FOLD_EVERY_MS = 5 * 60_000
|
||||
/** Between par2_files prunes (complete and judged sets' file lists). */
|
||||
const PRUNE_EVERY_MS = 60 * 60_000
|
||||
/** A prune's share of the merger's time: the rest goes on in the next one. */
|
||||
const PRUNE_BUDGET_MS = 2 * 60_000
|
||||
/** The merger's row in worker_cursors: the end of its last whole window, kept over a restart. */
|
||||
const CURSOR = 'merger'
|
||||
|
||||
export interface MergeRound {
|
||||
sets: number
|
||||
merges: number
|
||||
releases: number
|
||||
/** The name pass: merges, releases in them. */
|
||||
nameMerges: number
|
||||
nameReleases: number
|
||||
/** The inner-name pass: stems checked, merges, releases in them. */
|
||||
stems: number
|
||||
stemMerges: number
|
||||
stemReleases: number
|
||||
/** The sidecar fold: candidates looked at, sidecar-only ones among them, folded into their upload. */
|
||||
folded?: { looked: number; sidecars: number; folded: number }
|
||||
}
|
||||
|
||||
/**
|
||||
* A lone inspection's inner stem: its yEnc name up to the first dot, when 16–64 letters and digits
|
||||
* ("bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001"). The same expression as release_inspections_inner_stem_idx.
|
||||
*/
|
||||
const INNER_STEM = `substring(i.result->>'yencName' from '^([A-Za-z0-9]{16,64})\\.')`
|
||||
|
||||
/**
|
||||
* A file name as the name pass compares it: without its folder, case, spaces and punctuation, its UTF-8 read back when
|
||||
* it was read as Latin-1 ("Vermächtnis" posted, "Vermächtnis" in the par2).
|
||||
*/
|
||||
export function fileKey(name: string): string {
|
||||
let n = name.replace(/^.*[\\/]/, '')
|
||||
if (/[\u00c2-\u00f4][\u0080-\u00bf]/.test(n)) {
|
||||
const read = Buffer.from(n, 'latin1').toString('utf8')
|
||||
if (!read.includes('\ufffd')) n = read
|
||||
}
|
||||
return n.normalize('NFC').toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '')
|
||||
}
|
||||
|
||||
/** The releases holding a par2 file f (its first 16 KiB and size), none when it's a common file (COMMON_FILE). */
|
||||
const HOLDERS = `SELECT c.release_id FROM (
|
||||
SELECT l.release_id, count(*) OVER () AS holders
|
||||
FROM (SELECT h.release_id FROM file_hashes h WHERE h.hash16k = f.hash16k AND h.size = f.length LIMIT ${COMMON_FILE + 1}) l
|
||||
) c WHERE c.holders <= ${COMMON_FILE}`
|
||||
|
||||
interface Member {
|
||||
id: number
|
||||
group_id: number
|
||||
files: number
|
||||
posted_at: Date | null
|
||||
readable: boolean
|
||||
poster_id?: number | null
|
||||
/** The poster's text (joined from posters) where a query needs it. */
|
||||
poster?: string | null
|
||||
file_total?: number | null
|
||||
}
|
||||
|
||||
/**
|
||||
* The par2 merge (REBUILD.md, phase 2): releases whose files a par2 set
|
||||
* describes are one upload. Obfuscated uploads post each file under a name of
|
||||
* its own ("[9/20] - "14f5999a7bd4680b0720e524cac0b3b5"", hep: 13,232
|
||||
* one-file releases), so the header pass can't group them; the prober reads
|
||||
* each such file's first article (a `lone` candidate) and stores the MD5 of
|
||||
* its first 16 KiB, and a par2 body's file list holds the same hash per file,
|
||||
* whatever the file was renamed to. Releases in one group, posted within
|
||||
* MAX_SPREAD_MS of a release that holds the set's par2, become the one with
|
||||
* the most files: its chunks, probe candidates, inspections and hashes, the
|
||||
* sums of its counts, and when none of them has a readable name the name the
|
||||
* par2's file names share. Each merge is kept in release_merges.
|
||||
*
|
||||
* A second pass merges by inner name: the yEnc name in a lone file's first
|
||||
* article keeps the upload's random stem when the subject names each file
|
||||
* apart ("[1/7] - "471044d8…"" holds "bX147zx8I0oEgj7nzFCiorCT0ps2jEcb.7z.001";
|
||||
* hep, hephaestus, gyges: every stem under one counter total; LOG.md
|
||||
* 2026-10-12 (32)). Releases whose lone inspections share such a stem, in one
|
||||
* group, by one poster, under the same counter total and within MAX_SPREAD_MS,
|
||||
* are one upload. Only random stems: a readable one is grouped from the
|
||||
* headers already, and a repost would share it.
|
||||
*/
|
||||
export class Par2Merger {
|
||||
private stopped = false
|
||||
private folder = new SidecarFolder((trx, found, key, via) => this.merge(trx, found, key, via))
|
||||
/** Null until read from worker_cursors (1970 the first time: everything). */
|
||||
private since: Date | null = null
|
||||
|
||||
stop() {
|
||||
this.stopped = true
|
||||
}
|
||||
|
||||
async run() {
|
||||
let prunedAt = 0
|
||||
while (!this.stopped) {
|
||||
const started = Date.now()
|
||||
try {
|
||||
// First, at start too: a prune needn't wait for a long round.
|
||||
if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
|
||||
prunedAt = Date.now()
|
||||
const pruned = await new Par2Pruner().round(PRUNE_BUDGET_MS)
|
||||
if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged, ${pruned.final} final.`)
|
||||
}
|
||||
const round = await this.round()
|
||||
if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
|
||||
if (round.nameMerges > 0) logger.info(`Merged ${round.nameReleases} releases into ${round.nameMerges} by their par2 set's file names.`)
|
||||
if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
|
||||
if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
|
||||
} catch (error: any) {
|
||||
logger.error(`Par2 merge round failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The sets and inner stems with something new since the last round (a par2 found, a hash of one of their files, a
|
||||
* lone inspection), a window at a time.
|
||||
*/
|
||||
async round(): Promise<MergeRound> {
|
||||
// A Date keeps milliseconds, now() microseconds: rounded up, a row made at now() is before the end.
|
||||
const { rows: now } = await db.rawQuery(`SELECT now() + interval '1 millisecond' AS at`)
|
||||
const end = new Date(now[0].at)
|
||||
if (!this.since) {
|
||||
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
|
||||
this.since = saved ? new Date(saved.at) : new Date(0)
|
||||
}
|
||||
// First: sidecars (.nfo, .sfv, .nzb) apart from their upload join it. A queue, not since: it needn't wait for the
|
||||
// par2 pass (hours when that starts over).
|
||||
const folded = await this.folder.round()
|
||||
let foldedAt = Date.now()
|
||||
const foldAgain = async () => {
|
||||
if (Date.now() - foldedAt < FOLD_EVERY_MS) return
|
||||
const more = await this.folder.round()
|
||||
folded.looked += more.looked
|
||||
folded.sidecars += more.sidecars
|
||||
folded.folded += more.folded
|
||||
foldedAt = Date.now()
|
||||
if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
|
||||
}
|
||||
const round: MergeRound = { sets: 0, merges: 0, releases: 0, nameMerges: 0, nameReleases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
|
||||
while (this.since < end) {
|
||||
if (this.stopped) return round
|
||||
const from = await this.firstNew(new Date(this.since.getTime() - OVERLAP_MS), end)
|
||||
const until = from ? new Date(Math.min(from.getTime() + WINDOW_MS, end.getTime())) : end
|
||||
if (from) {
|
||||
const { touched, sets } = await this.setsToMerge(from, until)
|
||||
round.sets += touched
|
||||
for (const setId of sets) {
|
||||
if (this.stopped) return round
|
||||
await foldAgain()
|
||||
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
|
||||
const merged = await this.mergeSet(setId.trim()).catch((error) => {
|
||||
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.merges++
|
||||
round.releases += merged + 1
|
||||
}
|
||||
}
|
||||
// Every set the window found, by its files' names: readable files have no hashes (the files of one upload under
|
||||
// names that differ: scene tracks apart from their album's par2, "Vermaechtnis" beside "Vermächtnis"; LOG.md
|
||||
// 2026-10-23 (97)).
|
||||
const { rows: found } = await db.rawQuery(`SELECT DISTINCT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?`, [from, until])
|
||||
for (const { set_id: setId } of found) {
|
||||
if (this.stopped) return round
|
||||
const merged = await this.mergeByName(setId.trim()).catch((error) => {
|
||||
logger.error(`Name merge of set ${setId.trim()} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.nameMerges++
|
||||
round.nameReleases += merged + 1
|
||||
}
|
||||
}
|
||||
const { rows: stems } = await db.rawQuery(
|
||||
`SELECT DISTINCT ${INNER_STEM} AS stem FROM release_inspections i
|
||||
WHERE i.kind = 'lone' AND i.created_at >= ? AND i.created_at <= ? AND ${INNER_STEM} IS NOT NULL`,
|
||||
[from, until]
|
||||
)
|
||||
for (const { stem } of stems) {
|
||||
if (this.stopped) return round
|
||||
if (!looksRandom(stem)) continue
|
||||
round.stems++
|
||||
const merged = await this.mergeStem(stem).catch((error) => {
|
||||
logger.error(`Inner-name merge of ${stem} failed: ${error.message}`)
|
||||
return 0
|
||||
})
|
||||
if (merged > 0) {
|
||||
round.stemMerges++
|
||||
round.stemReleases += merged + 1
|
||||
}
|
||||
}
|
||||
}
|
||||
// Only after a whole window: a restart halfway goes on from the window before.
|
||||
this.since = until
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||
[CURSOR, this.since]
|
||||
)
|
||||
}
|
||||
return round
|
||||
}
|
||||
|
||||
/** When the first row a window reads was made, from `from` up to `end`; null when none (a gap, the nightly pause). */
|
||||
private async firstNew(from: Date, end: Date): Promise<Date | null> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT least(
|
||||
(SELECT min(created_at) FROM par2_sets WHERE created_at >= ?),
|
||||
(SELECT min(created_at) FROM file_hashes WHERE created_at >= ?),
|
||||
(SELECT min(created_at) FROM release_inspections WHERE created_at >= ?)) AS at`,
|
||||
[from, from, from]
|
||||
)
|
||||
const at = rows[0].at ? new Date(rows[0].at) : null
|
||||
return at && at <= end ? at : null
|
||||
}
|
||||
|
||||
/**
|
||||
* The sets a window touched (a par2 found, a hash of one of their files), and those among them a merge could
|
||||
* change: held or matched by two releases or more. Most have every file in one release already (96% of a backlog
|
||||
* hour's 147k): mergeSet needn't look at them.
|
||||
*/
|
||||
private async setsToMerge(from: Date, until: Date): Promise<{ touched: number; sets: string[] }> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`WITH touched AS MATERIALIZED (
|
||||
SELECT set_id FROM par2_sets WHERE created_at >= ? AND created_at <= ?
|
||||
UNION
|
||||
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size
|
||||
WHERE h.created_at >= ? AND h.created_at <= ?
|
||||
)
|
||||
SELECT (SELECT count(*) FROM touched) AS touched, array(
|
||||
SELECT t.set_id FROM touched t
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM (
|
||||
SELECT s.release_id FROM par2_sets s WHERE s.set_id = t.set_id
|
||||
UNION ALL
|
||||
SELECT m.release_id FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) m WHERE f.set_id = t.set_id
|
||||
) u HAVING count(DISTINCT u.release_id) >= 2)
|
||||
) AS sets`,
|
||||
[from, until, from, until]
|
||||
)
|
||||
return { touched: Number(rows[0].touched), sets: rows[0].sets }
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases a set's file names find (mergeByName without merging): its newest holder and the holders near it,
|
||||
* and the releases of its poster in its group posted within NAME_SPREAD_MS whose files the set all names, besides
|
||||
* sidecars, par2 and images. Null when there's nothing to merge. Not a holder further off: the same par2 posted
|
||||
* again years later ("Auto Bild Klassik Februar 02-2014": 2014 and 2026) is a repost (decision d).
|
||||
*/
|
||||
async nameMatches(trx: any, setId: string): Promise<Map<string, Member> | null> {
|
||||
const { rows: all } = await trx.rawQuery(
|
||||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, p.poster
|
||||
FROM par2_sets s JOIN releases r ON r.id = s.release_id LEFT JOIN posters p ON p.id = r.poster_id
|
||||
WHERE s.set_id = ? ORDER BY r.posted_at DESC NULLS LAST, r.id`,
|
||||
[setId]
|
||||
)
|
||||
const anchor: Member | undefined = all[0]
|
||||
// A random poster (one per file) bounds nothing: those uploads are the hash pass's.
|
||||
if (!anchor?.poster || !anchor.posted_at || randomPoster(anchor.poster)) return null
|
||||
const holders = all.filter(
|
||||
(h: Member) => h.group_id === anchor.group_id && h.posted_at && Math.abs(new Date(h.posted_at).getTime() - new Date(anchor.posted_at!).getTime()) <= NAME_SPREAD_MS
|
||||
)
|
||||
const { rows: described } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
|
||||
if (described.length === 0) return null
|
||||
const names = new Set(described.map((d: { name: string }) => fileKey(d.name)))
|
||||
const at = new Date(anchor.posted_at).getTime()
|
||||
const { rows: candidates } = await trx.rawQuery(
|
||||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id FROM releases r
|
||||
WHERE r.poster_id = ? AND r.group_id = ? AND r.posted_at BETWEEN ? AND ? AND r.id <> ALL(?)
|
||||
AND r.files <= ?
|
||||
ORDER BY r.posted_at DESC NULLS LAST, r.id DESC LIMIT ?`,
|
||||
// More files than the set describes, with its sidecars, can't all be named in it.
|
||||
[anchor.poster_id, anchor.group_id, new Date(at - NAME_SPREAD_MS), new Date(at + NAME_SPREAD_MS), holders.map((h: Member) => h.id), described.length + SLACK_FILES, NAME_CANDIDATES]
|
||||
)
|
||||
const found = new Map<string, Member>(holders.map((h: Member) => [String(h.id), h]))
|
||||
let files = holders.reduce((n: number, h: Member) => n + Number(h.files), 0)
|
||||
for (const c of candidates) {
|
||||
const own = await NzbService.files(Number(c.id), trx)
|
||||
const content = own.filter((f) => !besideSet(f.name))
|
||||
if (content.length === 0 || !content.every((f) => names.has(fileKey(f.name)))) continue
|
||||
found.set(String(c.id), c)
|
||||
files += Number(c.files)
|
||||
}
|
||||
if (found.size < 2) return null
|
||||
// No more files than the set describes, with its par2 volumes and sidecars (as the hash pass).
|
||||
if (files > MAX_FILES_PER_SET_FILE * names.size + SLACK_FILES) return null
|
||||
return found
|
||||
}
|
||||
|
||||
/** Merges the releases whose files one set names (nameMatches); how many went into another. */
|
||||
async mergeByName(setId: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const found = await this.nameMatches(trx, setId)
|
||||
return found ? this.merge(trx, found, setId, 'name') : 0
|
||||
})
|
||||
}
|
||||
|
||||
/** Merges the releases whose lone files' inner names share a random stem; how many went into another. */
|
||||
async mergeStem(stem: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT DISTINCT r.id, r.group_id, r.files, r.posted_at, r.readable, r.poster_id, r.file_total
|
||||
FROM release_inspections i JOIN releases r ON r.id = i.release_id
|
||||
WHERE i.kind = 'lone' AND ${INNER_STEM} = ?`,
|
||||
[stem]
|
||||
)
|
||||
const sorted: Member[] = rows.sort((a: Member, b: Member) => (a.posted_at ? new Date(a.posted_at).getTime() : 0) - (b.posted_at ? new Date(b.posted_at).getTime() : 0))
|
||||
const anchor = sorted[0]
|
||||
if (!anchor) return 0
|
||||
const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
|
||||
const total = (r: Member) => (r.file_total ? Number(r.file_total) : null)
|
||||
const near = (r: Member) =>
|
||||
r.group_id === anchor.group_id &&
|
||||
(r.poster_id ?? null) === (anchor.poster_id ?? null) &&
|
||||
(total(r) === null || total(anchor) === null || total(r) === total(anchor)) &&
|
||||
(at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
|
||||
const found = new Map<string, Member>()
|
||||
for (const r of sorted.filter(near)) found.set(String(r.id), r)
|
||||
if (found.size < 2) return 0
|
||||
// No more files than the counter says, with room for par2 volumes posted under it.
|
||||
const counted = Math.max(0, ...[...found.values()].map((r) => total(r) ?? 0))
|
||||
const files = [...found.values()].reduce((n, m) => n + Number(m.files), 0)
|
||||
if (counted > 0 && files > counted + SLACK_FILES) return 0
|
||||
return this.merge(trx, found, stem, 'inner')
|
||||
})
|
||||
}
|
||||
|
||||
/** Merges the releases of one set; how many went into another. */
|
||||
async mergeSet(setId: string): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const { rows: holders } = await trx.rawQuery(
|
||||
`SELECT r.id, r.group_id, r.files, r.posted_at, r.readable FROM par2_sets s JOIN releases r ON r.id = s.release_id
|
||||
WHERE s.set_id = ? ORDER BY r.id`,
|
||||
[setId]
|
||||
)
|
||||
const anchor: Member | undefined = holders[0]
|
||||
if (!anchor) return 0
|
||||
const at = (r: Member) => (r.posted_at ? new Date(r.posted_at).getTime() : null)
|
||||
const near = (r: Member) => r.group_id === anchor.group_id && (at(r) === null || at(anchor) === null || Math.abs(at(r)! - at(anchor)!) <= MAX_SPREAD_MS)
|
||||
const { rows: matches } = await trx.rawQuery(
|
||||
`SELECT f.file_id, r.id, r.group_id, r.files, r.posted_at, r.readable
|
||||
FROM par2_files f CROSS JOIN LATERAL (${HOLDERS}) h JOIN releases r ON r.id = h.release_id
|
||||
WHERE f.set_id = ?`,
|
||||
[setId]
|
||||
)
|
||||
// A file of the set that matches more than one release nearby is ambiguous: files that share their first
|
||||
// 16 KiB and size (131 uploads of 1 GiB test files "kantodoNNN.bin" matched one set's single file and became
|
||||
// one release of 1,284 files; LOG.md 2026-10-12 (16)). A plain repost of the same file stays apart too.
|
||||
const byFile = new Map<string, Map<string, Member>>()
|
||||
for (const m of matches.filter(near)) {
|
||||
const releases = byFile.get(m.file_id) ?? new Map<string, Member>()
|
||||
releases.set(String(m.id), m)
|
||||
byFile.set(m.file_id, releases)
|
||||
}
|
||||
const nearHolders = holders.filter(near)
|
||||
if (nearHolders.length > MAX_HOLDERS) return 0
|
||||
const found = new Map<string, Member>()
|
||||
for (const r of nearHolders) found.set(String(r.id), r)
|
||||
for (const releases of byFile.values()) if (releases.size === 1) for (const [id, r] of releases) found.set(id, r)
|
||||
if (found.size < 2) return 0
|
||||
const { rows: described } = await trx.rawQuery(`SELECT count(*) AS n FROM par2_files WHERE set_id = ?`, [setId])
|
||||
const setFiles = Math.max(Number(described[0].n), 1)
|
||||
if ([...found.values()].reduce((n, m) => n + Number(m.files), 0) > MAX_FILES_PER_SET_FILE * setFiles + SLACK_FILES) return 0
|
||||
return this.merge(trx, found, setId, 'par2')
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Merges the found releases into the one with the most files (named by the par2 set's files when none
|
||||
* is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
|
||||
*/
|
||||
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar' | 'name'): Promise<number> {
|
||||
{
|
||||
// Locked in id order (the batch store and other rounds take the same rows).
|
||||
const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
|
||||
const members: Member[] = locked.map((r: { id: string }) => found.get(String(r.id))!)
|
||||
if (members.length < 2) return 0
|
||||
const target = members.reduce((a, b) => (Number(b.files) > Number(a.files) ? b : a))
|
||||
const others = members.filter((m) => m.id !== target.id).map((m) => Number(m.id))
|
||||
const name = via === 'par2' && members.every((m) => !m.readable) ? await this.setName(trx, key) : undefined
|
||||
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO release_merges (from_id, from_key, into_id, set_id, files, via)
|
||||
SELECT id, encode(release_key, 'hex'), ?, ?, files, ? FROM releases WHERE id = ANY(?)`,
|
||||
[target.id, key, via, others]
|
||||
)
|
||||
// A candidate of an article the target, or another member of lower id, has already (the same file in two releases)
|
||||
// isn't moved: the name pass met them (every round of 3 sets failed on the unique key; LOG.md 2026-10-23 (99)).
|
||||
await trx.rawQuery(
|
||||
`DELETE FROM probe_candidates p WHERE p.release_id = ANY(?)
|
||||
AND (EXISTS (SELECT 1 FROM probe_candidates t WHERE t.release_id = ? AND t.kind = p.kind AND t.message_id = p.message_id)
|
||||
OR EXISTS (SELECT 1 FROM probe_candidates o WHERE o.release_id = ANY(?) AND o.release_id < p.release_id AND o.kind = p.kind AND o.message_id = p.message_id))`,
|
||||
[others, target.id, others]
|
||||
)
|
||||
for (const table of ['release_chunks', 'probe_candidates', 'release_inspections', 'file_hashes']) {
|
||||
await trx.rawQuery(`UPDATE ${table} SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
|
||||
}
|
||||
// An unsettled member's candidates into a settled target: the settler won't come by again to make them claimable.
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE r.id = ? AND r.settled_at IS NOT NULL AND pc.release_id = r.id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||
[target.id]
|
||||
)
|
||||
// The members' 7z tails together: those below the highest volume go.
|
||||
await pruneSevenZipTails(trx, [target.id])
|
||||
// A set held by the target already, or by two of the others (a par2 and its repost), isn't held twice (every
|
||||
// round failed on the primary key from 20:28 UTC on 2026-10-12; LOG.md 2026-10-12 (30)).
|
||||
await trx.rawQuery(
|
||||
`DELETE FROM par2_sets p WHERE p.release_id = ANY(?)
|
||||
AND (EXISTS (SELECT 1 FROM par2_sets t WHERE t.set_id = p.set_id AND t.release_id = ?)
|
||||
OR EXISTS (SELECT 1 FROM par2_sets o WHERE o.set_id = p.set_id AND o.release_id = ANY(?) AND o.release_id < p.release_id))`,
|
||||
[others, target.id, others]
|
||||
)
|
||||
await trx.rawQuery(`UPDATE par2_sets SET release_id = ? WHERE release_id = ANY(?)`, [target.id, others])
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases t SET
|
||||
files = t.files + s.files, segments = t.segments + s.segments, bytes = t.bytes + s.bytes, chunks = t.chunks + s.chunks,
|
||||
first_article = least(t.first_article, s.first_article), last_article = greatest(t.last_article, s.last_article),
|
||||
posted_at = least(t.posted_at, s.posted_at), last_posted_at = greatest(t.last_posted_at, s.last_posted_at),
|
||||
first_seen_at = least(t.first_seen_at, s.first_seen_at),
|
||||
readable = t.readable OR s.readable, has_par2 = t.has_par2 OR s.has_par2, passworded = t.passworded OR s.passworded,
|
||||
file_total = greatest(t.file_total, s.file_total), inner_name = coalesce(t.inner_name, s.inner_name),
|
||||
name = coalesce(nullif(?, ''), t.name)
|
||||
FROM (SELECT sum(files) AS files, sum(segments) AS segments, sum(bytes) AS bytes, sum(chunks) AS chunks,
|
||||
min(first_article) AS first_article, max(last_article) AS last_article, min(posted_at) AS posted_at,
|
||||
max(last_posted_at) AS last_posted_at, min(first_seen_at) AS first_seen_at,
|
||||
bool_or(readable) AS readable, bool_or(has_par2) AS has_par2, bool_or(passworded) AS passworded,
|
||||
max(file_total) AS file_total, max(inner_name) AS inner_name
|
||||
FROM releases WHERE id = ANY(?)) s
|
||||
WHERE t.id = ?`,
|
||||
[name ?? '', others, target.id]
|
||||
)
|
||||
// Completeness as the settler decides it, once settled; without a counter, by the par2 sets it now holds.
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases SET complete = CASE WHEN file_total IS NULL THEN complete ELSE files >= file_total END WHERE id = ? AND settled_at IS NOT NULL`,
|
||||
[target.id]
|
||||
)
|
||||
await trx.rawQuery(`DELETE FROM releases WHERE id = ANY(?)`, [others])
|
||||
await decidePar2Completeness(trx, [target.id])
|
||||
return others.length
|
||||
}
|
||||
}
|
||||
|
||||
/** The name a set's files share (par2ReleaseName), for an upload none of whose names is readable. */
|
||||
private async setName(trx: any, setId: string): Promise<string | undefined> {
|
||||
const { rows } = await trx.rawQuery(`SELECT name FROM par2_files WHERE set_id = ?`, [setId])
|
||||
const name = par2ReleaseName(rows.map((r: { name: string }) => r.name))
|
||||
// A par2 made after the renaming names the random names again.
|
||||
return name && !looksRandom(name) ? name.slice(0, 255) : undefined
|
||||
}
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
|
||||
/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
|
||||
const SETTLED_MS = 2 * 86400_000
|
||||
/** Sets per statement. */
|
||||
const BATCH = 20_000
|
||||
/** The sweep's row in worker_cursors: the last set_id it looked at (key); past the last set it starts over. */
|
||||
const CURSOR = 'par2-prune'
|
||||
/** The merger's row in worker_cursors: the end of its last whole window. */
|
||||
const MERGER = 'merger'
|
||||
/** The merger's windows look back this far (Par2Merger's OVERLAP_MS). */
|
||||
const MERGER_OVERLAP_MS = 60_000
|
||||
/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
|
||||
const JUNK = ['dead', 'fake', 'repeated']
|
||||
/** Probe candidates whose results a merge reads: a file's hash (lone), a set's file list (par2). */
|
||||
const MERGE_KINDS = ['lone', 'par2']
|
||||
|
||||
export interface PruneRound {
|
||||
complete: number
|
||||
judged: number
|
||||
final: number
|
||||
rows: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60), 2026-10-20 (87)). A set's file list
|
||||
* matches file hashes to releases and names an unreadable upload once; it's done when
|
||||
* - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
|
||||
* set (all of it merged into one);
|
||||
* - judged: every release holding it has a junk verdict (dead, fake, repeated);
|
||||
* - final: SETTLED_MS after its last holder's par2, every holder is final (release_final: a run of done ranges holds
|
||||
* it with hours to spare, so no more of its upload will be read), the candidates of the ranges around it are probed,
|
||||
* and the merger has seen every hash of its files. A file still missing then is one no probe will hash: most sets
|
||||
* describe a readable upload's files, which aren't lone candidates (9 in 10 sets were kept for that, 19 GB).
|
||||
* par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
|
||||
* par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
|
||||
* The sweep goes through the sets with rows by set_id and starts over at the end: a set kept now (a holder at the edge
|
||||
* of a run, its ranges' probes not done yet) is looked at again on the next pass.
|
||||
*/
|
||||
export class Par2Pruner {
|
||||
/** Sets per batch (BATCH; smaller in the tests). */
|
||||
constructor(private batch = BATCH) {}
|
||||
|
||||
/**
|
||||
* Batches until a pass is done or `budgetMs` is spent (the cursor is saved per batch, the next round goes on from
|
||||
* it): run from the merger's loop, a first round over two days of sets held the merges back for hours (LOG.md
|
||||
* 2026-10-17 (72)).
|
||||
*/
|
||||
async round(budgetMs = Infinity): Promise<PruneRound> {
|
||||
const deadline = Date.now() + budgetMs
|
||||
const round: PruneRound = { complete: 0, judged: 0, final: 0, rows: 0 }
|
||||
const { rows: cursors } = await db.rawQuery(`SELECT key FROM worker_cursors WHERE name = ?`, [CURSOR])
|
||||
let after: string = cursors[0]?.key ?? ''
|
||||
// Ranges done since the oldest merge-kind candidate still unprobed: their releases' hashes and file lists may be
|
||||
// still to come. Hashes since the merger's last window: it hasn't matched them yet.
|
||||
const { rows: marks } = await db.rawQuery(
|
||||
`SELECT coalesce((SELECT min(created_at) FROM probe_candidates WHERE status IN ('waiting', 'taken') AND kind = ANY(?)), now()) AS probed,
|
||||
coalesce((SELECT at FROM worker_cursors WHERE name = ?), to_timestamp(0)) - make_interval(secs => ?) AS merged,
|
||||
now() - make_interval(secs => ?) AS settled`,
|
||||
[MERGE_KINDS, MERGER, MERGER_OVERLAP_MS / 1000, SETTLED_MS / 1000]
|
||||
)
|
||||
const { probed, merged, settled } = marks[0]
|
||||
for (;;) {
|
||||
const { rows: sets } = await db.rawQuery(
|
||||
`WITH batch AS MATERIALIZED (SELECT DISTINCT set_id FROM par2_files WHERE set_id > ? ORDER BY set_id LIMIT ?),
|
||||
recent AS MATERIALIZED (SELECT group_id, posted_low, posted_high FROM ranges WHERE status = 'done' AND done_at >= ?),
|
||||
unseen AS MATERIALIZED (SELECT DISTINCT hash16k, size FROM file_hashes WHERE created_at >= ?)
|
||||
SELECT b.set_id, CASE
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = b.set_id AND NOT EXISTS (
|
||||
SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
|
||||
WHERE h.hash16k = f.hash16k AND h.size = f.length)) THEN 'complete'
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts v ON v.release_id = o.release_id AND v.rule = ANY(?)
|
||||
WHERE o.set_id = b.set_id AND v.release_id IS NULL) THEN 'judged'
|
||||
WHEN NOT EXISTS (SELECT 1 FROM par2_sets s WHERE s.set_id = b.set_id AND s.created_at > ?)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_sets s JOIN releases r ON r.id = s.release_id WHERE s.set_id = b.set_id AND (
|
||||
r.settled_at IS NULL
|
||||
OR EXISTS (SELECT 1 FROM recent x WHERE x.group_id = r.group_id
|
||||
AND x.posted_low <= coalesce(r.last_posted_at, r.posted_at) + interval '6 hours' AND x.posted_high >= r.posted_at - interval '6 hours')))
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_files f JOIN unseen u ON u.hash16k = f.hash16k AND u.size = f.length WHERE f.set_id = b.set_id)
|
||||
THEN 'final' END AS reason
|
||||
FROM batch b ORDER BY b.set_id`,
|
||||
[after, this.batch, probed, merged, settled, JUNK, settled]
|
||||
)
|
||||
const done = (reason: keyof PruneRound) => sets.filter((s: { reason: string }) => s.reason === reason).map((s: { set_id: string }) => s.set_id)
|
||||
for (const reason of ['complete', 'judged', 'final'] as const) {
|
||||
const ids = done(reason)
|
||||
round[reason] += ids.length
|
||||
// The holders' completeness while the file lists are there (app/services/par2_completeness.ts).
|
||||
if (reason !== 'judged' && ids.length > 0) {
|
||||
const { rows: holders } = await db.rawQuery(`SELECT DISTINCT release_id FROM par2_sets WHERE set_id = ANY(?::char(32)[])`, [ids])
|
||||
await decidePar2Completeness(db, holders.map((h: { release_id: string }) => h.release_id))
|
||||
}
|
||||
round.rows += await this.drop(ids)
|
||||
}
|
||||
// Past the last set: the next batch starts the next pass.
|
||||
after = sets.length < this.batch ? '' : sets.at(-1).set_id
|
||||
await db.rawQuery(
|
||||
`INSERT INTO worker_cursors (name, at, key) VALUES (?, now(), ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at, key = excluded.key`,
|
||||
[CURSOR, after]
|
||||
)
|
||||
if (after === '' || Date.now() > deadline) break
|
||||
}
|
||||
return round
|
||||
}
|
||||
|
||||
private async drop(setIds: string[]): Promise<number> {
|
||||
if (setIds.length === 0) return 0
|
||||
const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds])
|
||||
return result.rowCount ?? 0
|
||||
}
|
||||
}
|
||||
@@ -1,423 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { explored } from '#lib/rules/index'
|
||||
import { usableName, type Inspection } from '#lib/probe/index'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { twinKey, twinNames } from '#services/hash_twins'
|
||||
|
||||
/** pending_releases.status (migration 6400). */
|
||||
export const PENDING = {
|
||||
waiting: 0,
|
||||
named: 1,
|
||||
discarded: 2,
|
||||
exhausted: 3,
|
||||
naming: 4,
|
||||
taken: 9,
|
||||
} as const
|
||||
|
||||
/** The probe targets of a pending release in the order of effort: a par2 names the set's files outright, an archive's start its entries, a lone file only its yEnc name, a 7z's tail is the costliest find. */
|
||||
export const EFFORT = ['par2', 'archive', 'lone', '7z-tail'] as const
|
||||
|
||||
export interface PendingTarget {
|
||||
kind: string
|
||||
messageId: string
|
||||
file: string
|
||||
/** The target article's bytes (candidates from 2026-11-02 on). */
|
||||
bytes?: number
|
||||
}
|
||||
|
||||
/** One probe the ladder hands the prober: the next target of a pending release. */
|
||||
export interface PendingProbe {
|
||||
/** pending_releases.release_key as hex. */
|
||||
releaseKey: string
|
||||
kind: string
|
||||
messageId: string
|
||||
file: string
|
||||
tool: string
|
||||
groupId: number
|
||||
/** Its class (lib/release/classes.ts; 0 for a row from before the classes). */
|
||||
klass: number
|
||||
poster: string
|
||||
/** Targets probed before this one. */
|
||||
tries: number
|
||||
targets: number
|
||||
}
|
||||
|
||||
export type PendingOutcome = {
|
||||
probe: PendingProbe
|
||||
status: 'done' | 'missing' | 'error' | 'waiting'
|
||||
inspection?: Inspection
|
||||
}
|
||||
|
||||
export function byEffort(targets: PendingTarget[]): PendingTarget[] {
|
||||
const rank = (kind: string) => {
|
||||
const i = (EFFORT as readonly string[]).indexOf(kind)
|
||||
return i < 0 ? EFFORT.length : i
|
||||
}
|
||||
return [...targets].sort((a, b) => rank(a.kind) - rank(b.kind))
|
||||
}
|
||||
|
||||
/** A class's record (pending_classes), for the claim's order. */
|
||||
interface ClassRow {
|
||||
tool: string
|
||||
groupId: number
|
||||
klass: number
|
||||
probed: number
|
||||
named: number
|
||||
waiting: number
|
||||
}
|
||||
|
||||
/** The record is read again this often. */
|
||||
const CLASSES_MS = 60_000
|
||||
/** A class found short of a claim's share is left alone this long. */
|
||||
const EMPTY_MS = 5 * 60_000
|
||||
/** The prior rate of an unexplored class and its weight in probes: the ladder's overall rate on 2026-11-02 (LOG.md (130)). */
|
||||
const PRIOR_RATE = 0.15
|
||||
const PRIOR_WEIGHT = 10
|
||||
/** One class takes at most this share of a claim: the rest explores the classes after it. */
|
||||
const CLASS_SHARE = 0.25
|
||||
|
||||
/** The expected naming rate of a class: its record shrunk towards the prior. */
|
||||
export function classScore(c: { probed: number; named: number }): number {
|
||||
return (c.named + PRIOR_RATE * PRIOR_WEIGHT) / (c.probed + PRIOR_WEIGHT)
|
||||
}
|
||||
|
||||
/** An UPDATE returns its rows in no order: the claim's (the cheapest target, then the oldest) restored. */
|
||||
function claimed(rows: any[]): any[] {
|
||||
const bytes = (r: any) =>
|
||||
r.target_bytes === null ? Number.POSITIVE_INFINITY : Number(r.target_bytes)
|
||||
return [...rows].sort(
|
||||
(a, b) =>
|
||||
bytes(a) - bytes(b) ||
|
||||
new Date(a.created_at).getTime() - new Date(b.created_at).getTime() ||
|
||||
Number(a.first_article) - Number(b.first_article)
|
||||
)
|
||||
}
|
||||
|
||||
function toProbe(r: any): PendingProbe {
|
||||
const targets = byEffort(r.candidates as PendingTarget[])
|
||||
const tries = Number(r.tries)
|
||||
const target = targets[Math.min(tries, targets.length - 1)]
|
||||
return {
|
||||
releaseKey: r.release_key,
|
||||
kind: target.kind,
|
||||
messageId: target.messageId,
|
||||
file: target.file,
|
||||
tool: r.tool,
|
||||
groupId: Number(r.group_id),
|
||||
klass: Number(r.klass ?? 0),
|
||||
poster: r.poster,
|
||||
tries,
|
||||
targets: targets.length,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The pending ladder (REBUILD.md decision f): pending_releases probed from least to most effort, a usable name
|
||||
* handing the release to the namer (app/services/pending_namer.ts), no name after the last target exhausting it;
|
||||
* every finished climb counted on its signature (tool, group, poster; and the tool and group together) and on its
|
||||
* class (tool, group, size bucket × name shape × partial; lib/release/classes.ts, LOG.md (130)), and a signature
|
||||
* or class with enough probes and next to no names dropped: its waiting releases are discarded without a probe,
|
||||
* a sample of them (the rules' samplePercent) still climbing so the rate keeps being measured. The claim takes the
|
||||
* classes with the best record first (classScore), the cheapest target first within one.
|
||||
*/
|
||||
class PendingLadder {
|
||||
/** A previous prober's claims back in the queue. */
|
||||
async release(): Promise<number> {
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ?`,
|
||||
[PENDING.waiting, PENDING.taken]
|
||||
)
|
||||
return rowCount ?? 0
|
||||
}
|
||||
|
||||
/** The ladder's record per class, cached CLASSES_MS between claims, best first; classes found empty, until when. */
|
||||
private classes: ClassRow[] = []
|
||||
private classesAt = 0
|
||||
private empty = new Map<string, number>()
|
||||
|
||||
/** The record read again on the next claim (the tests). */
|
||||
forget() {
|
||||
this.classesAt = 0
|
||||
this.empty.clear()
|
||||
}
|
||||
|
||||
/** pending_classes with waiting releases and not dropped, by their expected rate (a prior for the unexplored). */
|
||||
private async classRows(): Promise<ClassRow[]> {
|
||||
if (Date.now() - this.classesAt < CLASSES_MS) return this.classes
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT tool, group_id, klass, probed, named, waiting FROM pending_classes WHERE waiting > 0 AND NOT dropped`
|
||||
)
|
||||
this.classes = rows
|
||||
.map((r: any) => ({
|
||||
tool: r.tool as string,
|
||||
groupId: Number(r.group_id),
|
||||
klass: Number(r.klass),
|
||||
probed: Number(r.probed),
|
||||
named: Number(r.named),
|
||||
waiting: Number(r.waiting),
|
||||
}))
|
||||
.sort((a: ClassRow, b: ClassRow) => classScore(b) - classScore(a))
|
||||
this.classesAt = Date.now()
|
||||
return this.classes
|
||||
}
|
||||
|
||||
/**
|
||||
* The next target of up to `n` waiting releases with a target left, claimed: by class, the best record first,
|
||||
* at most CLASS_SHARE of the claim from one class (the rest explores the classes after it), the cheapest target
|
||||
* and then the oldest within one; what the classes don't fill, the oldest of the rest (rows from before the
|
||||
* classes, status 0 of a class without a row).
|
||||
*/
|
||||
async claim(n: number): Promise<PendingProbe[]> {
|
||||
if (n <= 0) return []
|
||||
const out: PendingProbe[] = []
|
||||
const perClass = Math.max(1, Math.floor(n * CLASS_SHARE))
|
||||
const now = Date.now()
|
||||
for (const c of await this.classRows()) {
|
||||
if (out.length >= n) break
|
||||
const key = `${c.tool}\t${c.groupId}\t${c.klass}`
|
||||
if ((this.empty.get(key) ?? 0) > now) continue
|
||||
const want = Math.min(perClass, n - out.length)
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ? FROM (
|
||||
SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND tool = ? AND group_id = ? AND klass = ? AND finished AND tries < jsonb_array_length(candidates)
|
||||
ORDER BY target_bytes NULLS LAST, created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
WHERE p.release_key = c.release_key
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||||
[PENDING.taken, PENDING.waiting, c.tool, c.groupId, c.klass, want]
|
||||
)
|
||||
if (rows.length < want) this.empty.set(key, now + EMPTY_MS)
|
||||
if (rows.length > 0) {
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes SET waiting = greatest(0, waiting - ?) WHERE tool = ? AND group_id = ? AND klass = ?`,
|
||||
[rows.length, c.tool, c.groupId, c.klass]
|
||||
)
|
||||
}
|
||||
out.push(...claimed(rows).map(toProbe))
|
||||
}
|
||||
if (out.length < n) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE pending_releases p SET status = ? FROM (
|
||||
SELECT release_key FROM pending_releases
|
||||
WHERE status = ? AND finished AND tries < jsonb_array_length(candidates)
|
||||
ORDER BY created_at, first_article LIMIT ? FOR UPDATE SKIP LOCKED) c
|
||||
WHERE p.release_key = c.release_key
|
||||
RETURNING encode(p.release_key, 'hex') AS release_key, p.candidates, p.tries, p.tool, p.group_id, p.klass, coalesce(p.poster, '') AS poster, p.target_bytes, p.created_at, p.first_article`,
|
||||
[PENDING.taken, PENDING.waiting, n - out.length]
|
||||
)
|
||||
out.push(...claimed(rows).map(toProbe))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* What the probes showed: a usable name moves the release to the namer (status 4), else the next target, or the
|
||||
* end of the ladder (status 3); the finished climbs counted on their signatures.
|
||||
*/
|
||||
async record(outcomes: PendingOutcome[], trx: TransactionClientContract | typeof db = db) {
|
||||
if (outcomes.length === 0) return
|
||||
// A twin by content names what the probe itself didn't (hash_twins.ts).
|
||||
const twins = await twinNames(
|
||||
outcomes
|
||||
.filter((o) => o.status === 'done' && o.inspection?.hash16k && o.inspection.size)
|
||||
.map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
|
||||
trx
|
||||
)
|
||||
const rows = outcomes.map((o) => {
|
||||
const p = o.probe
|
||||
if (o.status === 'waiting') {
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.waiting,
|
||||
tries: p.tries,
|
||||
named: null,
|
||||
inspection: null,
|
||||
final: null,
|
||||
}
|
||||
}
|
||||
const probed = o.status === 'done' && o.inspection ? usableName(o.inspection, p.kind) : null
|
||||
const twin =
|
||||
o.status === 'done' && o.inspection?.hash16k
|
||||
? twins.get(twinKey(o.inspection.hash16k, o.inspection.size))
|
||||
: undefined
|
||||
const name = probed ?? twin ?? null
|
||||
const tries = p.tries + 1
|
||||
const inspection = {
|
||||
kind: p.kind,
|
||||
status: o.status,
|
||||
result: o.inspection ? { ...o.inspection, par2: undefined } : null,
|
||||
}
|
||||
if (name)
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.naming,
|
||||
tries: 0,
|
||||
named: name,
|
||||
inspection,
|
||||
final: 'named',
|
||||
}
|
||||
if (tries >= p.targets)
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.exhausted,
|
||||
tries,
|
||||
named: null,
|
||||
inspection,
|
||||
final: 'exhausted',
|
||||
}
|
||||
return {
|
||||
release_key: p.releaseKey,
|
||||
status: PENDING.waiting,
|
||||
tries,
|
||||
named: null,
|
||||
inspection,
|
||||
final: null,
|
||||
}
|
||||
})
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_releases p SET status = v.status, tries = v.tries, named = coalesce(v.named, p.named),
|
||||
inspections = CASE WHEN v.inspection IS NULL THEN p.inspections ELSE p.inspections || v.inspection END,
|
||||
probed_at = CASE WHEN v.inspection IS NULL THEN p.probed_at ELSE now() END
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(release_key text, status int, tries int, named text, inspection jsonb)
|
||||
WHERE p.release_key = decode(v.release_key, 'hex')`,
|
||||
[JSON.stringify(rows.map(({ final: _final, ...row }) => row))]
|
||||
)
|
||||
// The signatures: the poster's own and the tool's and group's together.
|
||||
const counts = new Map<
|
||||
string,
|
||||
{ tool: string; group_id: number; poster: string; probed: number; named: number }
|
||||
>()
|
||||
for (const [i, o] of outcomes.entries()) {
|
||||
const final = rows[i].final
|
||||
if (!final) continue
|
||||
for (const poster of [o.probe.poster, '']) {
|
||||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${poster}`
|
||||
const entry = counts.get(key) ?? {
|
||||
tool: o.probe.tool,
|
||||
group_id: o.probe.groupId,
|
||||
poster,
|
||||
probed: 0,
|
||||
named: 0,
|
||||
}
|
||||
entry.probed++
|
||||
if (final === 'named') entry.named++
|
||||
counts.set(key, entry)
|
||||
}
|
||||
}
|
||||
if (counts.size === 0) return
|
||||
const { pendingMinProbes, pendingMaxNamed } = coordinatorConfig.probe
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO pending_signatures (tool, group_id, poster, probed, named)
|
||||
SELECT v.tool, v.group_id, v.poster, v.probed, v.named FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text, probed int, named int)
|
||||
ON CONFLICT (tool, group_id, poster) DO UPDATE SET probed = pending_signatures.probed + excluded.probed, named = pending_signatures.named + excluded.named, updated_at = now()`,
|
||||
[JSON.stringify([...counts.values()])]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_signatures s SET dropped = (s.probed >= ?::int AND s.named::float <= s.probed * ?::float)
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, poster text)
|
||||
WHERE s.tool = v.tool AND s.group_id = v.group_id AND s.poster = v.poster`,
|
||||
[
|
||||
pendingMinProbes,
|
||||
pendingMaxNamed,
|
||||
JSON.stringify(
|
||||
[...counts.values()].map(({ tool, group_id, poster }) => ({ tool, group_id, poster }))
|
||||
),
|
||||
]
|
||||
)
|
||||
// The classes: the same record per (tool, group, class), and a probe given back is waiting again.
|
||||
const classes = new Map<
|
||||
string,
|
||||
{
|
||||
tool: string
|
||||
group_id: number
|
||||
klass: number
|
||||
probed: number
|
||||
named: number
|
||||
waiting: number
|
||||
}
|
||||
>()
|
||||
for (const [i, o] of outcomes.entries()) {
|
||||
const key = `${o.probe.tool}\t${o.probe.groupId}\t${o.probe.klass}`
|
||||
const entry = classes.get(key) ?? {
|
||||
tool: o.probe.tool,
|
||||
group_id: o.probe.groupId,
|
||||
klass: o.probe.klass,
|
||||
probed: 0,
|
||||
named: 0,
|
||||
waiting: 0,
|
||||
}
|
||||
const final = rows[i].final
|
||||
if (final) entry.probed++
|
||||
if (final === 'named') entry.named++
|
||||
if (o.status === 'waiting') entry.waiting++
|
||||
classes.set(key, entry)
|
||||
}
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO pending_classes (tool, group_id, klass, probed, named, waiting)
|
||||
SELECT v.tool, v.group_id, v.klass, v.probed, v.named, v.waiting FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, probed int, named int, waiting int)
|
||||
ON CONFLICT (tool, group_id, klass) DO UPDATE SET probed = pending_classes.probed + excluded.probed, named = pending_classes.named + excluded.named,
|
||||
waiting = pending_classes.waiting + excluded.waiting, updated_at = now()`,
|
||||
[JSON.stringify([...classes.values()])]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_classes c SET dropped = (c.probed >= ?::int AND c.named::float <= c.probed * ?::float)
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int)
|
||||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||||
[
|
||||
pendingMinProbes,
|
||||
pendingMaxNamed,
|
||||
JSON.stringify(
|
||||
[...classes.values()].map(({ tool, group_id, klass }) => ({ tool, group_id, klass }))
|
||||
),
|
||||
]
|
||||
)
|
||||
}
|
||||
|
||||
/** Waiting releases of a dropped signature discarded without a probe, the explored sample excepted; how many. */
|
||||
async discard(limit = 10_000): Promise<number> {
|
||||
const { samplePercent } = await RuleSets.current()
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT encode(p.release_key, 'hex') AS release_key FROM pending_releases p
|
||||
WHERE p.status = ? AND (EXISTS (
|
||||
SELECT 1 FROM pending_signatures s
|
||||
WHERE s.dropped AND s.tool = p.tool AND s.group_id = p.group_id AND (s.poster = '' OR s.poster = coalesce(p.poster, '')))
|
||||
OR EXISTS (SELECT 1 FROM pending_classes c WHERE c.dropped AND c.tool = p.tool AND c.group_id = p.group_id AND c.klass = p.klass))
|
||||
LIMIT ?`,
|
||||
[PENDING.waiting, limit]
|
||||
)
|
||||
const keys = rows
|
||||
.map((r: any) => r.release_key as string)
|
||||
.filter((key: string) => !explored(key, samplePercent))
|
||||
if (keys.length === 0) return 0
|
||||
const { rows: gone } = await db.rawQuery(
|
||||
`UPDATE pending_releases SET status = ? WHERE status = ? AND release_key = ANY(SELECT decode(k, 'hex') FROM unnest(?::text[]) AS k)
|
||||
RETURNING tool, group_id, klass`,
|
||||
[PENDING.discarded, PENDING.waiting, keys]
|
||||
)
|
||||
const counts = new Map<string, { tool: string; group_id: number; klass: number; n: number }>()
|
||||
for (const g of gone) {
|
||||
const key = `${g.tool}\t${g.group_id}\t${g.klass}`
|
||||
const entry = counts.get(key) ?? {
|
||||
tool: g.tool,
|
||||
group_id: Number(g.group_id),
|
||||
klass: Number(g.klass),
|
||||
n: 0,
|
||||
}
|
||||
entry.n++
|
||||
counts.set(key, entry)
|
||||
}
|
||||
if (counts.size > 0) {
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_classes c SET waiting = greatest(0, c.waiting - v.n)
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(tool text, group_id int, klass int, n int)
|
||||
WHERE c.tool = v.tool AND c.group_id = v.group_id AND c.klass = v.klass`,
|
||||
[JSON.stringify([...counts.values()])]
|
||||
)
|
||||
}
|
||||
return gone.length
|
||||
}
|
||||
}
|
||||
|
||||
export default new PendingLadder()
|
||||
@@ -1,229 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import { promisify } from 'node:util'
|
||||
import { unzip } from 'node:zlib'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { RangeIndexer, type ReleaseChunk } from '#lib/edge/index'
|
||||
import { identityKey } from '#lib/release/keys'
|
||||
import BatchStore from '#services/batch_store'
|
||||
import ChunkPartitions from '#services/chunk_partitions'
|
||||
import Corpus from '#services/corpus'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import { PENDING } from '#services/pending_ladder'
|
||||
|
||||
const inflate = promisify(unzip)
|
||||
|
||||
interface NamedRow {
|
||||
release_key: Buffer
|
||||
identity_key: Buffer
|
||||
group_id: number
|
||||
named: string
|
||||
first_article: number
|
||||
last_article: number
|
||||
file_total: number | null
|
||||
tries: number
|
||||
}
|
||||
|
||||
/** The namer gives a release up after this many failed moves (its articles gone from the server and the corpus). */
|
||||
const MAX_MOVES = 3
|
||||
|
||||
/**
|
||||
* The move into the catalog (REBUILD.md decision f): a pending release a probe named (status 4) gets its segments
|
||||
* read again, from the corpus when its range is stored, else from the server (XOVER over its article span, one
|
||||
* connection of its own, opened for it and closed after), through the TypeScript indexer, and is written as a
|
||||
* release with the name the probe found (source weak-stem, the name as inner name), its chunk and its probe
|
||||
* candidates, which the prober answers from probe_cache. Runs inside the prober.
|
||||
*/
|
||||
export class PendingNamer {
|
||||
private client: XoverClient | null = null
|
||||
private usedAt = 0
|
||||
|
||||
constructor(private corpusDir: string | undefined = coordinatorConfig.corpus.dir) {}
|
||||
|
||||
async run(stopped: () => boolean) {
|
||||
while (!stopped()) {
|
||||
let rows: NamedRow[] = []
|
||||
try {
|
||||
rows = await this.take(20)
|
||||
for (const row of rows) {
|
||||
if (stopped()) break
|
||||
try {
|
||||
await this.move(row)
|
||||
} catch (error: any) {
|
||||
logger.warn(
|
||||
`Naming ${row.release_key.toString('hex')} "${row.named}" failed: ${error.message}`
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE pending_releases SET tries = tries + 1, probed_at = now() WHERE release_key = ?`,
|
||||
[row.release_key]
|
||||
)
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Naming: ${error.message}`)
|
||||
}
|
||||
if (this.client && Date.now() - this.usedAt > coordinatorConfig.nntpIdleCloseMs) {
|
||||
this.client.close()
|
||||
this.client = null
|
||||
}
|
||||
if (rows.length === 0)
|
||||
for (let waited = 0; waited < 30_000 && !stopped(); waited += 1000) await sleep(1000)
|
||||
}
|
||||
this.client?.close()
|
||||
this.client = null
|
||||
}
|
||||
|
||||
/** Named releases awaiting their move, the longest waiting first; those failed MAX_MOVES times stay as they are. */
|
||||
async take(n: number): Promise<NamedRow[]> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT release_key, identity_key, group_id, named, first_article, last_article, file_total, tries
|
||||
FROM pending_releases WHERE status = ? AND tries < ? ORDER BY probed_at LIMIT ?`,
|
||||
[PENDING.naming, MAX_MOVES, n]
|
||||
)
|
||||
return rows.map((r: any) => ({
|
||||
...r,
|
||||
group_id: Number(r.group_id),
|
||||
first_article: Number(r.first_article),
|
||||
last_article: Number(r.last_article),
|
||||
tries: Number(r.tries),
|
||||
}))
|
||||
}
|
||||
|
||||
/** One named release into the catalog. */
|
||||
async move(row: NamedRow): Promise<void> {
|
||||
const group = await db.from('groups').where('id', row.group_id).firstOrFail()
|
||||
const range = await db
|
||||
.from('ranges')
|
||||
.where('group_id', row.group_id)
|
||||
.where('start_id', '<=', row.first_article)
|
||||
.where('end_id', '>=', row.first_article)
|
||||
.orderBy('id')
|
||||
.first()
|
||||
if (!range) throw new Error(`no range of ${group.name} holds article ${row.first_article}`)
|
||||
const text = await this.overviewText(group.name, row)
|
||||
const rules = await RuleSets.current()
|
||||
const ix = new RangeIndexer({
|
||||
group: group.name,
|
||||
start: row.first_article,
|
||||
end: row.last_article,
|
||||
rules: { ...rules, samplePercent: 0 },
|
||||
idleChunks: 0,
|
||||
edgeArticles: 0,
|
||||
// The edge wrote it partial when its file is short of parts on the server: so must the re-read.
|
||||
partialFiles: true,
|
||||
})
|
||||
ix.addText(text, row.last_article + 1)
|
||||
ix.finish()
|
||||
const batch = ix.take()
|
||||
const found = batch.releases.find((r) => identityKey(r.key).equals(row.identity_key))
|
||||
if (!found)
|
||||
throw new Error(
|
||||
`its articles ${row.first_article}–${row.last_article} of ${group.name} no longer show it`
|
||||
)
|
||||
// The catalog row under the pending key (the identity, the group, the first article the edge saw).
|
||||
const chunk: ReleaseChunk = {
|
||||
...found,
|
||||
name: row.named,
|
||||
source: 'weak-stem',
|
||||
readable: true,
|
||||
finished: true,
|
||||
edge: false,
|
||||
firstArticle: row.first_article,
|
||||
fileTotal: row.file_total ?? found.fileTotal,
|
||||
tool: undefined,
|
||||
}
|
||||
const candidates = batch.candidates
|
||||
.filter((c) => c.releaseKey === found.key)
|
||||
.map((c) => ({ ...c, upload: found.upload }))
|
||||
// The chunk's month partition, as the batch store makes sure of before its writes.
|
||||
await ChunkPartitions.ensure()
|
||||
await db.transaction(async (trx) => {
|
||||
const blockId = await BatchStore.blockOf(trx, range)
|
||||
await BatchStore.writeReleases(
|
||||
trx,
|
||||
{ groupId: row.group_id, blockId },
|
||||
row.first_article,
|
||||
[chunk],
|
||||
candidates
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases SET inner_name = ?, inspected_at = now() WHERE release_key = ? AND inner_name IS NULL`,
|
||||
[row.named.slice(0, 255), row.release_key]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE pending_releases SET status = ?, probed_at = now() WHERE release_key = ?`,
|
||||
[PENDING.named, row.release_key]
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/** The overview lines of the release's article span: the stored corpus chunks when its range is stored, else XOVER. */
|
||||
private async overviewText(groupName: string, row: NamedRow): Promise<string> {
|
||||
if (this.corpusDir) {
|
||||
const stored = await db
|
||||
.from('ranges')
|
||||
.where('group_id', row.group_id)
|
||||
.whereNotNull('stored_at')
|
||||
.where('start_id', '<=', row.last_article)
|
||||
.where('end_id', '>=', row.first_article)
|
||||
.orderBy('start_id')
|
||||
const parts: string[] = []
|
||||
let covered = true
|
||||
for (const r of stored) {
|
||||
const range = { group: groupName, start: Number(r.start_id), end: Number(r.end_id) }
|
||||
const size = coordinatorConfig.chunkArticles
|
||||
for (
|
||||
let from =
|
||||
range.start +
|
||||
Math.floor((Math.max(row.first_article, range.start) - range.start) / size) * size;
|
||||
from <= Math.min(row.last_article, range.end);
|
||||
from += size
|
||||
) {
|
||||
const raw = await Corpus.read(
|
||||
range,
|
||||
{ from, to: Math.min(from + size - 1, range.end) },
|
||||
this.corpusDir
|
||||
)
|
||||
if (!raw) {
|
||||
covered = false
|
||||
break
|
||||
}
|
||||
parts.push((await inflate(raw)).toString('latin1'))
|
||||
}
|
||||
if (!covered) break
|
||||
}
|
||||
if (
|
||||
covered &&
|
||||
parts.length > 0 &&
|
||||
stored.length > 0 &&
|
||||
Number(stored[0].start_id) <= row.first_article &&
|
||||
Number(stored[stored.length - 1].end_id) >= row.last_article
|
||||
) {
|
||||
return parts.join('')
|
||||
}
|
||||
}
|
||||
this.client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: true,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
this.usedAt = Date.now()
|
||||
try {
|
||||
await this.client.group(groupName)
|
||||
// One zlib/gzip stream, as the server sent it or deflated by the client (the corpus stores it so).
|
||||
const raw = await this.client.xoverRaw(row.first_article, row.last_article)
|
||||
return (await inflate(raw)).toString('latin1')
|
||||
} catch (error) {
|
||||
this.client.close()
|
||||
this.client = null
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,291 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import NntpPool from '#services/nntp_pool'
|
||||
|
||||
/**
|
||||
* The indexed groups and their ranges (REBUILD.md, group list, coordinator).
|
||||
* New groups get priorities after the existing ones (owner rule). A group's
|
||||
* span from GROUP is cut into ranges of RANGE_ARTICLES; a range's priority is
|
||||
* its position in its group's history (0 oldest … 1000 newest), so all groups
|
||||
* start with their oldest era. Planning again only adds ranges for new articles.
|
||||
*/
|
||||
class Planner {
|
||||
async addGroups(names: string[]): Promise<number> {
|
||||
let added = 0
|
||||
for (const name of names) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`INSERT INTO groups (name, priority) SELECT ?, coalesce(max(priority), 0) + 1 FROM groups ON CONFLICT (name) DO NOTHING RETURNING id`,
|
||||
[name]
|
||||
)
|
||||
added += rows.length
|
||||
}
|
||||
return added
|
||||
}
|
||||
|
||||
/** The `top` groups matching `pattern` by article span, largest first (LIST ACTIVE). */
|
||||
async largestGroups(pattern: string, top: number): Promise<string[]> {
|
||||
const groups = await NntpPool.use((client) => client.listActive(pattern))
|
||||
return groups
|
||||
.filter((g) => g.high > g.low)
|
||||
.sort((a, b) => b.high - b.low - (a.high - a.low))
|
||||
.slice(0, top)
|
||||
.map((g) => g.name)
|
||||
}
|
||||
|
||||
async planGroup(group: { id: number; name: string }): Promise<number> {
|
||||
const info = await NntpPool.use((client) => client.group(group.name))
|
||||
const last = await db.from('ranges').where('group_id', group.id).max('end_id as end').first()
|
||||
const rows = historyRows(group.id, info, Number(last?.end ?? 0), coordinatorConfig.rangeArticles)
|
||||
for (let i = 0; i < rows.length; i += 2000) await db.table('ranges').multiInsert(rows.slice(i, i + 2000))
|
||||
await db.from('groups').where('id', group.id).update({ low: info.low, high: info.high, planned_at: new Date() })
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/** History for groups not planned yet (new ones); what comes after is the tail's (planTail). */
|
||||
async planAll(): Promise<{ groups: number; ranges: number }> {
|
||||
const groups = await db.from('groups').where('active', true).whereNull('planned_at').orderBy('priority').select('id', 'name')
|
||||
let ranges = 0
|
||||
for (const group of groups) {
|
||||
// One group failing (e.g. gone from the server) doesn't stop the others.
|
||||
try {
|
||||
ranges += await this.planGroup({ id: Number(group.id), name: group.name })
|
||||
} catch (error: any) {
|
||||
logger.warn(`Planning ${group.name}: ${error.message}`)
|
||||
}
|
||||
}
|
||||
return { groups: groups.length, ranges }
|
||||
}
|
||||
|
||||
/**
|
||||
* The live tail: ranges for the articles after each planned group's last
|
||||
* range, at the tail's priority (before any history). The newest article
|
||||
* numbers come from LIST ACTIVE, one command per name prefix (GROUP per
|
||||
* group took minutes for 1000 groups).
|
||||
*/
|
||||
async planTail(now = new Date()): Promise<TailPlanned> {
|
||||
const groups = await db.from('groups').where('active', true).where('tail', true).whereNotNull('planned_at').select('id', 'name', 'planned_at')
|
||||
if (groups.length === 0) return { groups: 0, ranges: 0, extended: 0 }
|
||||
const { rows: ends } = await db.rawQuery(
|
||||
`SELECT DISTINCT ON (group_id) group_id, id, start_id, end_id,
|
||||
(status = 'planned' AND priority = ? AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL) AS open
|
||||
FROM ranges ORDER BY group_id, start_id DESC`,
|
||||
[coordinatorConfig.tail.priority]
|
||||
)
|
||||
const last = new Map<number, { id: number; start: number; end: number; open: boolean }>(
|
||||
ends.map((r: any) => [Number(r.group_id), { id: Number(r.id), start: Number(r.start_id), end: Number(r.end_id), open: r.open === true }])
|
||||
)
|
||||
const highs = new Map<string, number>()
|
||||
await NntpPool.use(async (client) => {
|
||||
for (const pattern of new Set(groups.map((g) => wildmatOf(g.name)))) {
|
||||
for (const g of await client.listActive(pattern)) highs.set(g.name, g.high)
|
||||
}
|
||||
})
|
||||
const plan = tailRanges(
|
||||
groups.map((g) => {
|
||||
const l = last.get(Number(g.id))
|
||||
return { id: Number(g.id), plannedTo: l?.end ?? null, plannedAt: new Date(g.planned_at), high: highs.get(g.name) ?? null, open: l?.open ? { id: l.id, start: l.start } : null }
|
||||
}),
|
||||
now,
|
||||
{ rangeArticles: coordinatorConfig.rangeArticles, ...coordinatorConfig.tail }
|
||||
)
|
||||
return this.applyTail(plan, now)
|
||||
}
|
||||
|
||||
/** The tail's plan written: the open ranges extended (one taken meanwhile gets the slice as a row instead), the rows added. */
|
||||
async applyTail(plan: TailPlan, now = new Date()): Promise<TailPlanned> {
|
||||
const rows = [...plan.rows]
|
||||
let extended = 0
|
||||
if (rows.length === 0 && plan.extended.length === 0) return { groups: 0, ranges: 0, extended: 0 }
|
||||
await db.transaction(async (trx) => {
|
||||
for (const e of plan.extended) {
|
||||
const { rows: done } = await trx.rawQuery(
|
||||
`UPDATE ranges SET end_id = ? WHERE id = ? AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL AND end_id = ? RETURNING id`,
|
||||
[e.end_id, e.id, e.from - 1]
|
||||
)
|
||||
if (done.length > 0) extended++
|
||||
else rows.push({ group_id: e.group_id, start_id: e.from, end_id: e.end_id, next_id: e.from, priority: coordinatorConfig.tail.priority })
|
||||
}
|
||||
if (rows.length > 0) await trx.table('ranges').multiInsert(rows)
|
||||
const highs = new Map<number, number>()
|
||||
for (const r of [...rows, ...plan.extended]) highs.set(r.group_id, Math.max(highs.get(r.group_id) ?? 0, r.end_id))
|
||||
for (const [id, high] of highs) await trx.from('groups').where('id', id).update({ high, planned_at: now })
|
||||
})
|
||||
return { groups: new Set([...rows, ...plan.extended].map((r) => r.group_id)).size, ranges: rows.length, extended }
|
||||
}
|
||||
|
||||
/**
|
||||
* Adjacent planned tail ranges of a group and priority, never spooled or taken, merged into ranges of at most
|
||||
* RANGE_ARTICLES (ranges:merge-tail): the tail planned a sliver a group every round while nothing leased the tail
|
||||
* (127k rows of a median 14k articles on 2026-10-27; LOG.md (118)). The first row of a run grows, the others go.
|
||||
*/
|
||||
async mergeTail(dryRun = false): Promise<{ candidates: number; merged: number; dropped: number }> {
|
||||
const { rows: candidates } = await db.rawQuery(
|
||||
`SELECT id, group_id, start_id, end_id, priority FROM ranges
|
||||
WHERE status = 'planned' AND priority < 0 AND NOT capture AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL
|
||||
ORDER BY group_id, start_id`
|
||||
)
|
||||
const merges = tailMerges(
|
||||
candidates.map((r: any) => ({ id: Number(r.id), group_id: Number(r.group_id), start_id: Number(r.start_id), end_id: Number(r.end_id), priority: Number(r.priority) })),
|
||||
coordinatorConfig.rangeArticles
|
||||
)
|
||||
const dropped = merges.reduce((n, m) => n + m.drop.length, 0)
|
||||
if (dryRun || merges.length === 0) return { candidates: candidates.length, merged: merges.length, dropped }
|
||||
await db.transaction(async (trx) => {
|
||||
// The lease picker reads several tables under this lock; the rows' guards catch a range taken meanwhile.
|
||||
await trx.rawQuery(`SELECT pg_advisory_xact_lock(hashtextextended('group-lease', 0))`)
|
||||
const { rows: gone } = await trx.rawQuery(
|
||||
`DELETE FROM ranges WHERE id = ANY(?::bigint[]) AND status = 'planned' AND spooled_chunks = 0 AND attempts = 0 AND node_id IS NULL RETURNING id`,
|
||||
[merges.flatMap((m) => m.drop)]
|
||||
)
|
||||
if (gone.length !== dropped) throw new Error(`${dropped - gone.length} of the ranges to drop were taken meanwhile; nothing merged`)
|
||||
const { rows: grown } = await trx.rawQuery(
|
||||
`UPDATE ranges r SET end_id = v.end_id FROM (SELECT unnest(?::bigint[]) AS id, unnest(?::bigint[]) AS end_id) v
|
||||
WHERE r.id = v.id AND r.status = 'planned' AND r.spooled_chunks = 0 AND r.attempts = 0 AND r.node_id IS NULL RETURNING r.id`,
|
||||
[merges.map((m) => m.id), merges.map((m) => m.end_id)]
|
||||
)
|
||||
if (grown.length !== merges.length) throw new Error(`${merges.length - grown.length} of the ranges to grow were taken meanwhile; nothing merged`)
|
||||
})
|
||||
return { candidates: candidates.length, merged: merges.length, dropped }
|
||||
}
|
||||
}
|
||||
|
||||
export interface TailPlanned {
|
||||
groups: number
|
||||
/** Rows added. */
|
||||
ranges: number
|
||||
/** Open ranges grown. */
|
||||
extended: number
|
||||
}
|
||||
|
||||
/** A LIST ACTIVE pattern for a group: its first two name parts (`alt.binaries.*`), or the name itself. */
|
||||
export function wildmatOf(name: string): string {
|
||||
const parts = name.split('.')
|
||||
return parts.length > 2 ? `${parts[0]}.${parts[1]}.*` : name
|
||||
}
|
||||
|
||||
export interface TailGroup {
|
||||
id: number
|
||||
/** End of its last range (null: none, then it has no tail yet). */
|
||||
plannedTo: number | null
|
||||
/** When its last range was planned. */
|
||||
plannedAt: Date
|
||||
/** The server's newest article (null: not listed). */
|
||||
high: number | null
|
||||
/**
|
||||
* Its last range when the tail may grow it instead of adding a row: a planned range at the tail's priority that
|
||||
* nothing spooled, took or tried (its start: the cut at rangeArticles counts from there).
|
||||
*/
|
||||
open?: { id: number; start: number } | null
|
||||
}
|
||||
|
||||
export interface TailRow {
|
||||
group_id: number
|
||||
start_id: number
|
||||
end_id: number
|
||||
next_id: number
|
||||
priority: number
|
||||
}
|
||||
|
||||
export interface TailPlan {
|
||||
/** New ranges. */
|
||||
rows: TailRow[]
|
||||
/** Open ranges grown to end_id (from: the first new article, the old end + 1). */
|
||||
extended: { id: number; group_id: number; from: number; end_id: number }[]
|
||||
}
|
||||
|
||||
/**
|
||||
* The tail's new ranges: per group, the articles after its last range, once there are `minArticles` of them or
|
||||
* the group's last planning is `maxDelayMinutes` ago. An open last range grows up to `rangeArticles` first, so a
|
||||
* tail nothing leases stays one range a group instead of a sliver a round; the rest is cut at `rangeArticles`.
|
||||
*/
|
||||
export function tailRanges(
|
||||
groups: TailGroup[],
|
||||
now: Date,
|
||||
options: { rangeArticles: number; minArticles: number; maxDelayMinutes: number; priority: number }
|
||||
): TailPlan {
|
||||
const rows: TailRow[] = []
|
||||
const extended: TailPlan['extended'] = []
|
||||
for (const g of groups) {
|
||||
if (g.plannedTo === null || g.high === null || g.high <= g.plannedTo) continue
|
||||
const waiting = g.high - g.plannedTo
|
||||
const waited = (now.getTime() - g.plannedAt.getTime()) / 60_000
|
||||
if (waiting < options.minArticles && waited < options.maxDelayMinutes) continue
|
||||
let start = g.plannedTo + 1
|
||||
if (g.open && g.open.start + options.rangeArticles - 1 >= start) {
|
||||
const end = Math.min(g.open.start + options.rangeArticles - 1, g.high)
|
||||
extended.push({ id: g.open.id, group_id: g.id, from: start, end_id: end })
|
||||
start = end + 1
|
||||
}
|
||||
for (; start <= g.high; start += options.rangeArticles) {
|
||||
rows.push({ group_id: g.id, start_id: start, end_id: Math.min(start + options.rangeArticles - 1, g.high), next_id: start, priority: options.priority })
|
||||
}
|
||||
}
|
||||
return { rows, extended }
|
||||
}
|
||||
|
||||
export interface TailMerge {
|
||||
/** The run's first range, grown to end_id. */
|
||||
id: number
|
||||
end_id: number
|
||||
/** The ranges it absorbed. */
|
||||
drop: number[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs of adjacent ranges of one group and priority (sorted by group and start) packed into ranges of at most
|
||||
* `rangeArticles`: the first range of each packed run grows, the others are dropped. Only runs that merge anything.
|
||||
*/
|
||||
export function tailMerges(
|
||||
ranges: { id: number; group_id: number; start_id: number; end_id: number; priority: number }[],
|
||||
rangeArticles: number
|
||||
): TailMerge[] {
|
||||
const merges: TailMerge[] = []
|
||||
let cur: (TailMerge & { group_id: number; start_id: number; priority: number }) | null = null
|
||||
const flush = () => {
|
||||
if (cur && cur.drop.length > 0) merges.push({ id: cur.id, end_id: cur.end_id, drop: cur.drop })
|
||||
}
|
||||
for (const r of ranges) {
|
||||
if (cur && r.group_id === cur.group_id && r.priority === cur.priority && r.start_id === cur.end_id + 1 && r.end_id - cur.start_id + 1 <= rangeArticles) {
|
||||
cur.end_id = r.end_id
|
||||
cur.drop.push(r.id)
|
||||
continue
|
||||
}
|
||||
flush()
|
||||
cur = { id: r.id, group_id: r.group_id, start_id: r.start_id, end_id: r.end_id, priority: r.priority, drop: [] }
|
||||
}
|
||||
flush()
|
||||
return merges
|
||||
}
|
||||
|
||||
/**
|
||||
* The rows a group's history is cut into, from `lastEnd + 1` (0: the group has no ranges yet) up to the high
|
||||
* watermark: RANGE_ARTICLES slices at the position priority. A group planned for the first time also gets one
|
||||
* "ancient" range, articles 1 … low − 1: what had expired on the server before it was planned. Status
|
||||
* 'ancient', never leased or fetched; it keeps the tiling complete from article 1 (migration 6700 added it for
|
||||
* the groups planned before 2026-10-28).
|
||||
*/
|
||||
export function historyRows(
|
||||
groupId: number,
|
||||
info: { low: number; high: number },
|
||||
lastEnd: number,
|
||||
size: number
|
||||
): Record<string, unknown>[] {
|
||||
const rows: Record<string, unknown>[] = []
|
||||
if (lastEnd === 0 && info.low > 1)
|
||||
rows.push({ group_id: groupId, start_id: 1, end_id: info.low - 1, next_id: 1, priority: 0, status: 'ancient' })
|
||||
const from = Math.max(info.low, lastEnd + 1)
|
||||
const span = Math.max(1, info.high - info.low)
|
||||
for (let start = from; start <= info.high; start += size) {
|
||||
rows.push({
|
||||
group_id: groupId,
|
||||
start_id: start,
|
||||
end_id: Math.min(start + size - 1, info.high),
|
||||
next_id: start,
|
||||
priority: Math.floor((1000 * (start - info.low)) / span),
|
||||
})
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
export default new Planner()
|
||||
@@ -1,610 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import NntpPool from '#services/nntp_pool'
|
||||
import Spool, { type ChunkSpan } from '#services/spool'
|
||||
import Corpus, { type CorpusFilter } from '#services/corpus'
|
||||
import RuleSets from '#services/rule_sets'
|
||||
import CatalogSpace from '#services/catalog_space'
|
||||
import { hedge, type Request, type XoverClient } from '#lib/nntp/index'
|
||||
|
||||
interface Target {
|
||||
id: number
|
||||
group: string
|
||||
/** Chunk starts to fetch without compression (ranges.plain_chunks). */
|
||||
plain: Set<number>
|
||||
/** Chunks not spooled yet, in order. */
|
||||
missing: ChunkSpan[]
|
||||
/** All of the range's chunks, from its start. */
|
||||
total: number
|
||||
start: number
|
||||
end: number
|
||||
/** Its chunks are kept in the corpus (ranges.capture), and read from there when stored. */
|
||||
capture: boolean
|
||||
/** Fetched into the corpus only, not the spool: a capture range no worker leases soon (corpus.ahead). */
|
||||
corpusOnly?: boolean
|
||||
}
|
||||
|
||||
/** A request's bytes in are sampled this often; STALL_SAMPLES of them make the minute a stall is judged over. */
|
||||
const STALL_SAMPLE_MS = 15_000
|
||||
const STALL_SAMPLES = 60_000 / STALL_SAMPLE_MS
|
||||
|
||||
/** Connection ages the report counts slow first requests by, in minutes. */
|
||||
const AGE_BUCKETS = [2, 10, Infinity]
|
||||
|
||||
/** How often the targets are read again from the database. */
|
||||
const REFRESH_MS = 2000
|
||||
/** How often the spool's size is measured (a stat per file). */
|
||||
const SIZE_MS = 30_000
|
||||
|
||||
/**
|
||||
* Keeps the news server busy ahead of the workers (REBUILD.md, prefetch).
|
||||
* Every connection takes the next missing chunk: the leased ranges' first
|
||||
* (a worker waits for them), then the next planned ranges, with at most
|
||||
* PREFETCH_PER_RANGE connections on a range and PREFETCH_PER_GROUP on a
|
||||
* group (Eweka answers faster spread out), about SPOOL_AHEAD_PER_LEASE per
|
||||
* active lease (at least SPOOL_MIN_AHEAD), within SPOOL_MAX_GB. One
|
||||
* connection fills ~26 k overview lines/s, so a range on one connection
|
||||
* kept its worker waiting (LOG.md 2026-10-09). A chunk that takes long is
|
||||
* asked again on a spare connection (fetchHedged).
|
||||
*/
|
||||
export class Prefetcher {
|
||||
/**
|
||||
* This process's share of the ranges, `id % count = index` (`coordinator:run --shard`): one
|
||||
* process's event loop handles ~60 MB/s of compressed XOVER (16.5% of a core at ~10 MB/s;
|
||||
* LOG.md 2026-10-09), less than 50 connections bring.
|
||||
*/
|
||||
constructor(private shard = { index: 0, count: 1 }) {}
|
||||
|
||||
private targets: Target[] = []
|
||||
private claimed = new Set<string>()
|
||||
/** Chunks being fetched per range and per group (spool.perRange, spool.perGroup). */
|
||||
private perRange = new Map<number, number>()
|
||||
private perGroup = new Map<string, number>()
|
||||
private running = 0
|
||||
private refreshedAt = 0
|
||||
private stopped = false
|
||||
private spoolBytes = 0
|
||||
private sizedAt = 0
|
||||
/** Fetched since the last report: chunks, articles, ms spent fetching. */
|
||||
private stats = this.freshStats()
|
||||
/** Loops in a fetch or a spool write right now. */
|
||||
private fetchingNow = 0
|
||||
private writingNow = 0
|
||||
private sampler?: NodeJS.Timeout
|
||||
/** Requests in flight: what, since when, on which connection (the report names the oldest). */
|
||||
private inFlight = new Set<{
|
||||
rangeId: number
|
||||
from: number
|
||||
to: number
|
||||
since: number
|
||||
client: XoverClient
|
||||
}>()
|
||||
/** Capture ranges found complete in the corpus (not looked at again). */
|
||||
private stored = new Set<number>()
|
||||
|
||||
private freshStats() {
|
||||
return {
|
||||
chunks: 0,
|
||||
/** Chunks read from the corpus instead of fetched. */
|
||||
stored: 0,
|
||||
/** Chunks fetched into the corpus only (corpus.ahead). */
|
||||
captured: 0,
|
||||
articles: 0,
|
||||
fetchMs: 0,
|
||||
hedged: 0,
|
||||
hedgeWon: 0,
|
||||
/** First requests per connection-age bucket, and how many of them were slow. */
|
||||
byAge: AGE_BUCKETS.map(() => ({ requests: 0, slow: 0 })),
|
||||
/** Spool writes and the range's update, ms. */
|
||||
writeMs: 0,
|
||||
/** Per answered request, ms: GROUP, XOVER until its status line, the transfer; transfers with a silence over 5 s. */
|
||||
requests: 0,
|
||||
groupMs: 0,
|
||||
replyMs: 0,
|
||||
bodyMs: 0,
|
||||
silent: 0,
|
||||
/** Once a second: loops fetching, writing, not running; pool connections with a command in flight. */
|
||||
samples: 0,
|
||||
fetching: 0,
|
||||
writing: 0,
|
||||
idle: 0,
|
||||
inFlight: 0,
|
||||
inFlightMin: Infinity,
|
||||
/** Requests given up as stalled (nntpStallBytesPerMinute). */
|
||||
deadlines: 0,
|
||||
ranges: 0,
|
||||
groups: 0,
|
||||
since: Date.now(),
|
||||
}
|
||||
}
|
||||
|
||||
/** The prefetcher's loops: the pool less the connections kept for hedging. */
|
||||
private get loops() {
|
||||
return Math.max(1, NntpPool.size - coordinatorConfig.nntpHedgeConnections)
|
||||
}
|
||||
|
||||
/** One round: refreshes the targets now and then, and puts idle connections to work. */
|
||||
async tick() {
|
||||
this.sampler ??= setInterval(() => this.sample(), 1000).unref()
|
||||
if (Date.now() - this.refreshedAt >= REFRESH_MS) {
|
||||
this.refreshedAt = Date.now()
|
||||
this.targets = await this.readTargets()
|
||||
}
|
||||
this.report()
|
||||
while (!this.stopped && this.running < this.loops && this.peek()) {
|
||||
this.running++
|
||||
this.work()
|
||||
.catch((error) => logger.warn(`Prefetching: ${error.message}`))
|
||||
.finally(() => this.running--)
|
||||
}
|
||||
// Work left with every loop running: the pool's share could be bigger.
|
||||
if (!this.stopped && this.running >= this.loops && this.peek()) NntpPool.starved()
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.stopped = true
|
||||
clearInterval(this.sampler)
|
||||
}
|
||||
|
||||
get active() {
|
||||
return this.running
|
||||
}
|
||||
|
||||
/** One connection's loop: chunks until there is nothing left to fetch. */
|
||||
private async work() {
|
||||
while (!this.stopped) {
|
||||
// The pool's share got smaller: fewer loops (waiting on the pool, they'd count as work short of a connection).
|
||||
// Two leaving at once is one too many for a tick (TICK_MS), whose loop comes back.
|
||||
if (this.running > this.loops) return
|
||||
const next = this.take()
|
||||
if (!next) return
|
||||
const { target, span } = next
|
||||
try {
|
||||
const fetchedAt = Date.now()
|
||||
this.fetchingNow++
|
||||
const fetch = async (client: Parameters<Parameters<typeof NntpPool.use>[0]>[0]) => {
|
||||
await client.group(target.group)
|
||||
return client.xoverRaw(span.from, span.to)
|
||||
}
|
||||
const corpusRange = { group: target.group, start: target.start, end: target.end }
|
||||
// A range in the corpus is read from disk (a reset re-indexes it without the news server).
|
||||
let data =
|
||||
target.capture && !target.corpusOnly ? await Corpus.read(corpusRange, span) : null
|
||||
const stored = data !== null
|
||||
try {
|
||||
if (!data)
|
||||
data = target.plain.has(span.from)
|
||||
? await NntpPool.usePlain(fetch)
|
||||
: await this.fetchHedged(target.id, target.group, span)
|
||||
} finally {
|
||||
this.fetchingNow--
|
||||
}
|
||||
if (stored) {
|
||||
this.stats.stored++
|
||||
} else {
|
||||
this.stats.chunks++
|
||||
this.stats.articles += span.to - span.from + 1
|
||||
this.stats.fetchMs += Date.now() - fetchedAt
|
||||
if (target.capture)
|
||||
await Corpus.write(corpusRange, span, data, target.total, await this.corpusFilter())
|
||||
}
|
||||
if (target.corpusOnly) {
|
||||
this.stats.captured++
|
||||
continue
|
||||
}
|
||||
const writtenAt = Date.now()
|
||||
this.writingNow++
|
||||
try {
|
||||
const existed = await Spool.has(target.id, span)
|
||||
await Spool.write(target.id, span, data)
|
||||
if (existed) continue
|
||||
this.spoolBytes += data.length
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE ranges SET spooled_chunks = spooled_chunks + 1 WHERE id = ? RETURNING spooled_chunks`,
|
||||
[target.id]
|
||||
)
|
||||
if (Number(rows[0]?.spooled_chunks) >= target.total)
|
||||
await db.from('ranges').where('id', target.id).update({ spooled_at: new Date() })
|
||||
} finally {
|
||||
this.writingNow--
|
||||
this.stats.writeMs += Date.now() - writtenAt
|
||||
}
|
||||
} finally {
|
||||
this.claimed.delete(`${target.id}:${span.from}`)
|
||||
this.count(target, -1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A chunk over a pooled connection. Not in after nntpHedgeAfterMs, it's
|
||||
* asked again on a free connection (if there is one): Eweka stalls single
|
||||
* requests, and the same chunk on another connection mostly comes in
|
||||
* seconds (LOG.md 2026-10-09 (15)). The first answer wins; the other
|
||||
* request can't be cancelled, so its connection is closed (the pool opens
|
||||
* a fresh one).
|
||||
*/
|
||||
private async fetchHedged(rangeId: number, group: string, span: ChunkSpan): Promise<Buffer> {
|
||||
let age: number | undefined
|
||||
const request = (
|
||||
start: (work: (client: XoverClient) => Promise<Buffer>) => Promise<Buffer> | null,
|
||||
first: boolean
|
||||
): Request<Buffer> | null => {
|
||||
let client: XoverClient | undefined
|
||||
let cancelled = false
|
||||
const done = start(async (c) => {
|
||||
// Given up before it got a connection: the connection goes back unused.
|
||||
if (cancelled) return Buffer.alloc(0)
|
||||
client = c
|
||||
if (first) age = NntpPool.ageOf(c)
|
||||
const entry = { rangeId, from: span.from, to: span.to, since: Date.now(), client: c }
|
||||
this.inFlight.add(entry)
|
||||
// Bytes in, sampled every STALL_SAMPLE_MS: less than nntpStallBytesPerMinute over the last minute is a stall.
|
||||
let late = false
|
||||
const samples: number[] = []
|
||||
const watch = setInterval(() => {
|
||||
samples.push(c.buffered)
|
||||
if (samples.length <= STALL_SAMPLES) return
|
||||
const minuteAgo = samples.shift()!
|
||||
if (c.buffered - minuteAgo < coordinatorConfig.nntpStallBytesPerMinute) {
|
||||
late = true
|
||||
this.stats.deadlines++
|
||||
c.close()
|
||||
}
|
||||
}, STALL_SAMPLE_MS)
|
||||
let data: Buffer
|
||||
let groupMs = 0
|
||||
try {
|
||||
const groupAt = performance.now()
|
||||
await c.group(group)
|
||||
groupMs = performance.now() - groupAt
|
||||
data = await c.xoverRaw(span.from, span.to)
|
||||
} catch (error) {
|
||||
if (!late) throw error
|
||||
throw new Error(
|
||||
`Chunk ${span.from}–${span.to} of range ${rangeId} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB in a minute, ${Math.round(c.buffered / 1000)} kB in); fetching it again`
|
||||
)
|
||||
} finally {
|
||||
clearInterval(watch)
|
||||
this.inFlight.delete(entry)
|
||||
}
|
||||
if (!cancelled) {
|
||||
const t = c.lastTiming
|
||||
this.stats.requests++
|
||||
this.stats.groupMs += groupMs
|
||||
this.stats.replyMs += t.replyMs
|
||||
this.stats.bodyMs += t.bodyMs
|
||||
if (t.longestGapMs > 5000) this.stats.silent++
|
||||
}
|
||||
return data
|
||||
})
|
||||
if (!done) return null
|
||||
done.catch(() => {})
|
||||
return {
|
||||
done,
|
||||
// The request can't be called off on the wire: its connection is closed, and the pool opens a fresh one.
|
||||
cancel: () => {
|
||||
cancelled = true
|
||||
client?.close()
|
||||
},
|
||||
}
|
||||
}
|
||||
const result = await hedge(
|
||||
() => request((work) => NntpPool.use(work), true)!,
|
||||
() => request((work) => NntpPool.tryUse(work), false),
|
||||
coordinatorConfig.nntpHedgeAfterMs
|
||||
)
|
||||
if (age !== undefined) {
|
||||
const bucket = this.stats.byAge[AGE_BUCKETS.findIndex((minutes) => age! < minutes * 60_000)]
|
||||
bucket.requests++
|
||||
if (result.late) bucket.slow++
|
||||
}
|
||||
if (result.hedged) this.stats.hedged++
|
||||
if (result.secondWon) this.stats.hedgeWon++
|
||||
return result.value
|
||||
}
|
||||
|
||||
/** The longest-running request: range, chunk, how long, how much came in. */
|
||||
private oldest() {
|
||||
let oldest:
|
||||
{ rangeId: number; from: number; to: number; since: number; client: XoverClient } | undefined
|
||||
for (const r of this.inFlight) if (!oldest || r.since < oldest.since) oldest = r
|
||||
if (!oldest) return 'none in flight'
|
||||
return `oldest request ${Math.round((Date.now() - oldest.since) / 1000)} s (range ${oldest.rangeId}, ${oldest.from}–${oldest.to}, ${Math.round(oldest.client.buffered / 1000)} kB in)`
|
||||
}
|
||||
|
||||
private sample() {
|
||||
const s = this.stats
|
||||
s.samples++
|
||||
s.fetching += this.fetchingNow
|
||||
s.writing += this.writingNow
|
||||
s.idle += Math.max(0, this.loops - this.fetchingNow - this.writingNow)
|
||||
s.inFlight += NntpPool.busyCount
|
||||
s.inFlightMin = Math.min(s.inFlightMin, NntpPool.busyCount)
|
||||
s.ranges += this.perRange.size
|
||||
s.groups += this.perGroup.size
|
||||
}
|
||||
|
||||
/** A line a minute: what the connections fetched, and how long a chunk took. */
|
||||
private report() {
|
||||
const seconds = (Date.now() - this.stats.since) / 1000
|
||||
if (seconds < 60) return
|
||||
const { chunks, articles, fetchMs, hedged, hedgeWon, byAge, samples, requests } = this.stats
|
||||
if (chunks > 0) {
|
||||
const ages = byAge
|
||||
.map((b, i) =>
|
||||
`${i === 0 ? '<' : `${AGE_BUCKETS[i - 1]}–`}${AGE_BUCKETS[i] === Infinity ? '' : AGE_BUCKETS[i]} min ${b.slow}/${b.requests}`.replace(
|
||||
'– min',
|
||||
'+ min'
|
||||
)
|
||||
)
|
||||
.join(', ')
|
||||
logger.info(
|
||||
`Prefetched ${chunks} chunks in ${seconds.toFixed(0)} s: ${Math.round(articles / seconds).toLocaleString('en')} articles/s, ` +
|
||||
`${(fetchMs / chunks / 1000).toFixed(1)} s a chunk; ` +
|
||||
`hedged ${hedged} (second won ${hedgeWon}); slow by connection age: ${ages}` +
|
||||
(this.stats.stored > 0 ? `; ${this.stats.stored} chunks from the corpus` : '') +
|
||||
(this.stats.captured > 0 ? `; ${this.stats.captured} fetched into the corpus only` : '') +
|
||||
'.'
|
||||
)
|
||||
const st = this.stats
|
||||
const avg = (n: number) => (samples > 0 ? n / samples : 0)
|
||||
const per = (ms: number) => (requests > 0 ? (ms / requests / 1000).toFixed(1) : '-')
|
||||
logger.info(
|
||||
`Connections: ${avg(st.inFlight).toFixed(1)} of ${NntpPool.size} with a command in flight (min ${st.inFlightMin}), ` +
|
||||
`on ${avg(st.ranges).toFixed(1)} ranges of ${avg(st.groups).toFixed(1)} groups; ` +
|
||||
`loops ${avg(st.fetching).toFixed(1)} fetching, ${avg(st.writing).toFixed(1)} writing, ${avg(st.idle).toFixed(1)} idle; ` +
|
||||
`a request: GROUP ${per(st.groupMs)} s, XOVER reply ${per(st.replyMs)} s, transfer ${per(st.bodyMs)} s; ` +
|
||||
`${st.silent} of ${requests} transfers silent >5 s; writing ${(st.writeMs / Math.max(1, chunks)).toFixed(0)} ms a chunk; ` +
|
||||
`${st.deadlines} stalled (under ${coordinatorConfig.nntpStallBytesPerMinute / 1000} kB a minute); ${this.oldest()}.`
|
||||
)
|
||||
}
|
||||
this.stats = this.freshStats()
|
||||
}
|
||||
|
||||
private peek(): boolean {
|
||||
return this.targets.some(
|
||||
(t) => this.open(t) && t.missing.some((s) => !this.claimed.has(`${t.id}:${s.from}`))
|
||||
)
|
||||
}
|
||||
|
||||
/** Whether the target is under its range's and its group's limit of connections. */
|
||||
private open(target: Target) {
|
||||
const { perRange, perGroup } = coordinatorConfig.spool
|
||||
return (
|
||||
(perRange <= 0 || (this.perRange.get(target.id) ?? 0) < perRange) &&
|
||||
(perGroup <= 0 || (this.perGroup.get(target.group) ?? 0) < perGroup)
|
||||
)
|
||||
}
|
||||
|
||||
private count(target: Target, by: number) {
|
||||
const range = (this.perRange.get(target.id) ?? 0) + by
|
||||
const group = (this.perGroup.get(target.group) ?? 0) + by
|
||||
if (range > 0) this.perRange.set(target.id, range)
|
||||
else this.perRange.delete(target.id)
|
||||
if (group > 0) this.perGroup.set(target.group, group)
|
||||
else this.perGroup.delete(target.group)
|
||||
}
|
||||
|
||||
/** The first unclaimed missing chunk, in target order, of a target under its limits. */
|
||||
private take(): { target: Target; span: ChunkSpan } | null {
|
||||
for (const target of this.targets) {
|
||||
if (!this.open(target)) continue
|
||||
const index = target.missing.findIndex((s) => !this.claimed.has(`${target.id}:${s.from}`))
|
||||
if (index === -1) continue
|
||||
const [span] = target.missing.splice(index, 1)
|
||||
this.claimed.add(`${target.id}:${span.from}`)
|
||||
this.count(target, 1)
|
||||
return { target, span }
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private async readTargets(): Promise<Target[]> {
|
||||
if (Date.now() - this.sizedAt >= SIZE_MS) {
|
||||
this.sizedAt = Date.now()
|
||||
this.spoolBytes = await Spool.size()
|
||||
}
|
||||
const mine = (q: any) =>
|
||||
this.shard.count > 1 ? q.whereRaw('r.id % ? = ?', [this.shard.count, this.shard.index]) : q
|
||||
const rows = (q: any) =>
|
||||
mine(
|
||||
q
|
||||
.from('ranges as r')
|
||||
.join('groups as g', 'g.id', 'r.group_id')
|
||||
.select(
|
||||
'r.id',
|
||||
'g.name as group',
|
||||
'r.start_id',
|
||||
'r.end_id',
|
||||
'r.next_id',
|
||||
'r.plain_chunks',
|
||||
'r.capture'
|
||||
)
|
||||
)
|
||||
// Leased ranges, checked against the spool's files whatever spooled_at says: a worker waits for them.
|
||||
const leased = await rows(db.query()).where('r.status', 'leased').orderBy('r.leased_at')
|
||||
// The groups workers hold (group_leases; REBUILD.md design 1), by lane: the prefetcher runs ahead of their
|
||||
// cursors (a group's history and tail lanes are two cursors).
|
||||
const held: { group: number; lane: string }[] = (
|
||||
await db
|
||||
.from('group_leases')
|
||||
.whereNotNull('node_id')
|
||||
.where(
|
||||
'heartbeat_at',
|
||||
'>=',
|
||||
new Date(Date.now() - coordinatorConfig.leaseTtlSeconds * 1000)
|
||||
)
|
||||
.select('group_id', 'lane')
|
||||
).map((l) => ({ group: Number(l.group_id), lane: l.lane }))
|
||||
// Planned ranges ahead: those started already, then new ones up to SPOOL_AHEAD_PER_LEASE per lease.
|
||||
const ahead = Math.max(
|
||||
coordinatorConfig.spool.minAhead,
|
||||
coordinatorConfig.spool.aheadPerLease * Math.max(leased.length, held.length)
|
||||
)
|
||||
const started = await rows(db.query())
|
||||
.where('r.status', 'planned')
|
||||
.where('r.spooled_chunks', '>', 0)
|
||||
.whereNull('r.spooled_at')
|
||||
.orderBy([{ column: 'r.priority' }, { column: 'r.id' }])
|
||||
const [{ ready }] = await mine(
|
||||
db.from('ranges as r').where('r.status', 'planned').where('r.spooled_chunks', '>', 0)
|
||||
).count('* as ready')
|
||||
const room = this.spoolBytes < coordinatorConfig.spool.maxBytes
|
||||
// With at most perRange connections a range, enough ranges still being fetched for all loops: fully
|
||||
// spooled ones count as ready but need no connection (30 of 40 busy on small ranges; LOG.md 2026-10-10 (2)).
|
||||
// 8 more than that: ranges with a chunk or two left take fewer than perRange (with +2, 3 of 40 loops idle).
|
||||
const { perRange } = coordinatorConfig.spool
|
||||
const short =
|
||||
perRange > 0 ? Math.ceil(this.loops / perRange) + 8 - (leased.length + started.length) : 0
|
||||
const want = Math.max(ahead - Number(ready), short)
|
||||
// The catalog at its cap on the NVMe: no history fetched ahead (CatalogSpace).
|
||||
const full = await CatalogSpace.full()
|
||||
// Ahead of each held cursor: the lane's next ranges in article order (the history lane's within what the group
|
||||
// leases hand out: LEASE_MIN/MAX_PRIORITY, LEASE_CAPTURE_ONLY, and none at the catalog's cap; the tail lane's
|
||||
// at the tail's priority), a range of every held cursor before any cursor's second; without held groups (no
|
||||
// window worker running) nothing is fetched ahead for workers.
|
||||
const { leasePriorityMin, leasePriorityMax, leaseCaptureOnly } = coordinatorConfig
|
||||
const tailPriority = coordinatorConfig.tail.priority
|
||||
const heldHistory = full ? [] : held.filter((h) => h.lane === 'history').map((h) => h.group)
|
||||
const heldTail = held.filter((h) => h.lane === 'tail').map((h) => h.group)
|
||||
const fresh =
|
||||
room && want > 0 && heldHistory.length + heldTail.length > 0
|
||||
? await rows(db.query())
|
||||
.where('r.status', 'planned')
|
||||
.where('r.spooled_chunks', 0)
|
||||
.whereRaw('r.next_id <= r.end_id')
|
||||
.where((q: any) =>
|
||||
q
|
||||
.where((h: any) =>
|
||||
h
|
||||
.whereIn('r.group_id', heldHistory)
|
||||
.where('r.priority', '>', tailPriority)
|
||||
.whereBetween('r.priority', [leasePriorityMin, leasePriorityMax])
|
||||
.where((c: any) => (leaseCaptureOnly ? c.where('r.capture', true) : c))
|
||||
)
|
||||
.orWhere((t: any) =>
|
||||
t.whereIn('r.group_id', heldTail).where('r.priority', '<=', tailPriority)
|
||||
)
|
||||
)
|
||||
.orderByRaw(
|
||||
'row_number() OVER (PARTITION BY r.group_id, r.priority <= ? ORDER BY r.start_id), r.priority, r.start_id',
|
||||
[tailPriority]
|
||||
)
|
||||
.limit(want)
|
||||
: []
|
||||
const targets: Target[] = []
|
||||
for (const row of [...leased, ...(room ? started : []), ...fresh]) {
|
||||
const start = Number(row.start_id)
|
||||
const end = Number(row.end_id)
|
||||
const total = Spool.chunksOf({ start, end }).length
|
||||
const missing: ChunkSpan[] = []
|
||||
// A chunk already in the corpus isn't spooled: the stream serves it from there (REBUILD.md design 1).
|
||||
const corpusRange = row.capture === true ? { group: row.group as string, start, end } : null
|
||||
for (const span of Spool.chunksOf({ start, end }, Number(row.next_id))) {
|
||||
if (this.claimed.has(`${row.id}:${span.from}`) || (await Spool.has(Number(row.id), span)))
|
||||
continue
|
||||
if (corpusRange && (await Corpus.has(corpusRange, span))) continue
|
||||
missing.push(span)
|
||||
}
|
||||
if (missing.length > 0) {
|
||||
targets.push({
|
||||
id: Number(row.id),
|
||||
group: row.group,
|
||||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||||
missing,
|
||||
total,
|
||||
start,
|
||||
end,
|
||||
capture: row.capture === true,
|
||||
})
|
||||
}
|
||||
}
|
||||
// Last, for loops with nothing else to do: capture ranges not stored yet, into the corpus only.
|
||||
for (const target of await this.corpusTargets(rows, new Set(targets.map((t) => t.id))))
|
||||
targets.push(target)
|
||||
return targets
|
||||
}
|
||||
|
||||
private filterRules: { at: number; filter: CorpusFilter } | null = null
|
||||
|
||||
/** The rules corpus chunks are filtered with (CORPUS_FILTER), read again once a minute; undefined when off. */
|
||||
private async corpusFilter(): Promise<CorpusFilter | undefined> {
|
||||
if (!coordinatorConfig.corpus.filter) return undefined
|
||||
if (!this.filterRules || Date.now() - this.filterRules.at > 60_000) {
|
||||
const rules = await RuleSets.current()
|
||||
this.filterRules = {
|
||||
at: Date.now(),
|
||||
filter: { ruleVersion: rules.version, priorities: rules.priorities },
|
||||
}
|
||||
}
|
||||
return this.filterRules.filter
|
||||
}
|
||||
|
||||
/**
|
||||
* Up to corpus.ahead capture ranges whose chunks aren't all in the corpus, by priority: planned ones not started
|
||||
* (not leased soon), and done ones (a window of every group, ranges:capture, takes those indexed already too);
|
||||
* as many of a group as its connections allow (perGroup / perRange): in id order the 16 were one group's, which
|
||||
* took 4 of the loops and left the rest idle (30k articles/s; LOG.md 2026-10-23 (100)). The candidates are taken
|
||||
* a range of every group at a time (the n-th unstored range of each group before any group's n+1-th): a group's
|
||||
* ranges have consecutive ids, and once the small groups of a window were stored the 512 lowest ids were all one
|
||||
* group's, two targets on two connections (10 ranges an hour; LOG.md 2026-10-25 (110)). A range found complete
|
||||
* gets stored_at and isn't read again.
|
||||
*/
|
||||
private async corpusTargets(rows: (q: any) => any, taken: Set<number>): Promise<Target[]> {
|
||||
const { dir, ahead } = coordinatorConfig.corpus
|
||||
if (!dir || ahead <= 0) return []
|
||||
const { perRange, perGroup } = coordinatorConfig.spool
|
||||
const ofGroup =
|
||||
perGroup > 0 && perRange > 0 ? Math.max(1, Math.floor(perGroup / perRange)) : Infinity
|
||||
const groups = new Map<string, number>()
|
||||
const candidates = await rows(db.query())
|
||||
.where('r.capture', true)
|
||||
// A group taken off the list (groups.active) isn't worth the connections.
|
||||
.where('g.active', true)
|
||||
.whereNull('r.stored_at')
|
||||
.where((q: any) =>
|
||||
q
|
||||
.where((p: any) => p.where('r.status', 'planned').where('r.spooled_chunks', 0))
|
||||
.orWhere('r.status', 'done')
|
||||
)
|
||||
.orderByRaw('r.priority, row_number() OVER (PARTITION BY r.group_id ORDER BY r.id), r.id')
|
||||
.limit(ahead * 16)
|
||||
const targets: Target[] = []
|
||||
for (const row of candidates) {
|
||||
if (targets.length >= ahead) break
|
||||
const id = Number(row.id)
|
||||
if (taken.has(id) || this.stored.has(id)) continue
|
||||
if ((groups.get(row.group) ?? 0) >= ofGroup) continue
|
||||
const start = Number(row.start_id)
|
||||
const end = Number(row.end_id)
|
||||
const range = { group: row.group, start, end }
|
||||
if (await Corpus.complete(range)) {
|
||||
this.stored.add(id)
|
||||
await db.from('ranges').where('id', id).update({ stored_at: new Date() })
|
||||
continue
|
||||
}
|
||||
const chunks = Spool.chunksOf({ start, end })
|
||||
const missing: ChunkSpan[] = []
|
||||
for (const span of chunks)
|
||||
if (!this.claimed.has(`${id}:${span.from}`) && !(await Corpus.has(range, span)))
|
||||
missing.push(span)
|
||||
if (missing.length > 0) {
|
||||
targets.push({
|
||||
id,
|
||||
group: row.group,
|
||||
plain: new Set((row.plain_chunks ?? []).map(Number)),
|
||||
missing,
|
||||
total: chunks.length,
|
||||
start,
|
||||
end,
|
||||
capture: true,
|
||||
corpusOnly: true,
|
||||
})
|
||||
groups.set(row.group, (groups.get(row.group) ?? 0) + 1)
|
||||
}
|
||||
}
|
||||
return targets
|
||||
}
|
||||
}
|
||||
@@ -1,457 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { YencError } from '#lib/yenc/index'
|
||||
import { inspect, innerName, PROBE_VERSION, type Inspection } from '#lib/probe/index'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import { DiskSpace } from '#services/disk_space'
|
||||
import { decidePar2Completeness } from '#services/par2_completeness'
|
||||
import { twinKey, twinNames } from '#services/hash_twins'
|
||||
import PendingLadder, { type PendingProbe } from '#services/pending_ladder'
|
||||
import { PendingNamer } from '#services/pending_namer'
|
||||
|
||||
interface Candidate {
|
||||
id: number
|
||||
releaseId: number
|
||||
kind: string
|
||||
messageId: string
|
||||
/** A pending release's target (the pending ladder), not a catalog candidate: id and releaseId are 0. */
|
||||
pending?: PendingProbe
|
||||
}
|
||||
|
||||
type Outcome = { candidate: Candidate; status: 'done' | 'missing' | 'error' | 'waiting'; inspection?: Inspection; cached?: boolean }
|
||||
|
||||
/**
|
||||
* The order candidates are probed in: what the par2 merge needs first (a random-named file's own start, then the
|
||||
* par2 sets and archives of releases without a readable name), then the rest (names and passwords of readable ones),
|
||||
* then 7z listings (the end of a split 7z's last volume; queued from 2026-10-13, probed from 2026-10-23).
|
||||
*/
|
||||
const TIERS: { kind: string; unreadable: boolean }[] = [
|
||||
{ kind: 'lone', unreadable: false },
|
||||
{ kind: 'par2', unreadable: true },
|
||||
{ kind: 'archive', unreadable: true },
|
||||
{ kind: 'par2', unreadable: false },
|
||||
{ kind: 'archive', unreadable: false },
|
||||
{ kind: '7z-tail', unreadable: true },
|
||||
{ kind: '7z-tail', unreadable: false },
|
||||
]
|
||||
const FLUSH_MS = 2000
|
||||
/** Between sweeps that discard the waiting releases of dropped signatures. */
|
||||
const SWEEP_MS = 5 * 60_000
|
||||
|
||||
/**
|
||||
* JSON for a jsonb parameter, without NUL characters: jsonb can't hold \u0000 (a name read from an article had one),
|
||||
* and the whole flush failed with it (LOG.md 2026-10-18 (78)).
|
||||
*/
|
||||
function jsonb(value: unknown): string {
|
||||
return JSON.stringify(value, (_key, v) => (typeof v === 'string' ? v.replaceAll('\0', '') : v))
|
||||
}
|
||||
|
||||
/** Rounds of every lane's pipeline still queued when the next claim starts. */
|
||||
const LOW_ROUNDS = 8
|
||||
|
||||
/**
|
||||
* The candidate queue's levels: the next claim starts below `low`, LOW_ROUNDS rounds of every lane's pipeline (at
|
||||
* ~0.2 s an article and a pipeline of 8, ~13 s of work), and fills it to `target`, twice that. A claim takes 1.5–4 s
|
||||
* and more (random slices of an 11 M-row table); refilling at half of four rounds left 3–6 s, and at 20–24 lanes the
|
||||
* lanes waited for candidates up to 19% of their time (LOG.md 2026-10-28 (120)). At least `minBatch` either way.
|
||||
*/
|
||||
export function queueLevels(lanes: number, pipeline: number, minBatch: number): { low: number; target: number } {
|
||||
const low = Math.max(minBatch, lanes * pipeline * LOW_ROUNDS)
|
||||
return { low, target: 2 * low }
|
||||
}
|
||||
|
||||
/** Slices a claim takes from random points of the queue. */
|
||||
const SLICES = 20
|
||||
const REPORT_MS = 60_000
|
||||
|
||||
/**
|
||||
* Phase 2's first step (REBUILD.md, part 6): reads the probe queue's
|
||||
* articles (settled releases first, by kind) over PROBE_CONNECTIONS
|
||||
* connections of its own, without compression, and stores what they show:
|
||||
* per release and kind an Inspection, and on the release whether it's
|
||||
* passworded and its inner name. One process: fetching (~0.1 s an article)
|
||||
* dwarfs parsing (a few ms), so there's no queue between them.
|
||||
*/
|
||||
export class Prober {
|
||||
private queue: Candidate[] = []
|
||||
private outcomes: Outcome[] = []
|
||||
private refilling: Promise<void> | null = null
|
||||
private stopped = false
|
||||
private stats = Prober.freshStats()
|
||||
/** Lanes running (each one connection, once it has work); at most the share's grant. Counted apart from the set: a lane leaving takes itself off at once. */
|
||||
private lanes = new Set<Promise<void>>()
|
||||
private running = 0
|
||||
/** The last lane to look found nothing to claim: the prober wants its minimum only. */
|
||||
private dry = false
|
||||
/** What it stores (inspections, par2 file lists, the WAL) fills the NVMe: no claims below PROBE_MIN_FREE_GB. */
|
||||
private space = new DiskSpace('probing', '/', coordinatorConfig.probe.minFreeGb)
|
||||
private share = new NntpShare('prober', coordinatorConfig.nntpShares.proberPriority, coordinatorConfig.probe.minConnections, coordinatorConfig.probe.connections)
|
||||
|
||||
/** Per minute: what was read, and where the lanes' time went (ms summed over lanes; flushes ms on their own). */
|
||||
private static freshStats() {
|
||||
return { articles: 0, bytes: 0, done: 0, missing: 0, error: 0, cached: 0, pending: 0, discarded: 0, waitMs: 0, idleMs: 0, fetchMs: 0, inspectMs: 0, flushMs: 0, claimMs: 0, since: Date.now() }
|
||||
}
|
||||
/** The pending ladder's namer: named pending releases into the catalog, alongside the lanes. */
|
||||
private namer = new PendingNamer()
|
||||
private sweptAt = 0
|
||||
|
||||
async run() {
|
||||
// A previous run's claims: one prober at a time.
|
||||
const { rowCount } = await db.rawQuery(`UPDATE probe_candidates SET status = 'waiting' WHERE status = 'taken'`)
|
||||
if (rowCount) logger.info(`Probing: ${rowCount} candidates from a previous run back in the queue.`)
|
||||
const released = await PendingLadder.release()
|
||||
if (released) logger.info(`Probing: ${released} pending releases from a previous run back on the ladder.`)
|
||||
const namer = this.namer.run(() => this.stopped).catch((error) => logger.error(`Naming failed: ${error.message}`))
|
||||
logger.info(`Probing with ${this.share.min}–${this.share.max} connections (its share of the account's).`)
|
||||
const flusher = (async () => {
|
||||
while (!this.stopped || this.outcomes.length > 0) {
|
||||
await sleep(FLUSH_MS)
|
||||
const flushed = Date.now()
|
||||
await this.flush().catch((error) => logger.error(`Probing: storing results failed: ${error.message}`))
|
||||
this.stats.flushMs += Date.now() - flushed
|
||||
this.report()
|
||||
}
|
||||
})()
|
||||
// Lanes as the share allows: the coordinator's history comes first (app/services/nntp_shares.ts).
|
||||
while (!this.stopped) {
|
||||
try {
|
||||
const was = this.share.granted
|
||||
const granted = await this.share.sync(this.dry ? 0 : this.share.max, this.running)
|
||||
if (granted !== was) logger.info(`Probing: ${granted} connections (was ${was}).`)
|
||||
while (this.running < granted) {
|
||||
this.running++
|
||||
const lane: Promise<void> = this.lane()
|
||||
.catch((error) => logger.error(`Probing: a lane failed: ${error.message}`))
|
||||
.finally(() => this.lanes.delete(lane))
|
||||
this.lanes.add(lane)
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Probing: NNTP share failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = 0; waited < coordinatorConfig.nntpShares.everyMs && !this.stopped; waited += 1000) await sleep(1000)
|
||||
}
|
||||
await Promise.all(this.lanes)
|
||||
await namer
|
||||
await this.share.leave().catch(() => {})
|
||||
// What was claimed and not probed goes back.
|
||||
for (const candidate of this.queue.splice(0)) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
this.stopped = true
|
||||
await flusher
|
||||
}
|
||||
|
||||
stop() {
|
||||
this.stopped = true
|
||||
}
|
||||
|
||||
/**
|
||||
* One connection's loop: PROBE_PIPELINE articles at a time (pipelined), reconnecting after a failure; done when
|
||||
* there are more lanes than the share (its connection closed, the coordinator takes it on its next sync).
|
||||
*/
|
||||
private async lane() {
|
||||
let client: XoverClient | null = null
|
||||
let usedAt = Date.now()
|
||||
try {
|
||||
while (!this.stopped) {
|
||||
if (this.running > this.share.granted) break
|
||||
const batch: Candidate[] = []
|
||||
const waited = Date.now()
|
||||
while (batch.length < coordinatorConfig.probe.pipeline) {
|
||||
const candidate = await this.next()
|
||||
if (!candidate) break
|
||||
batch.push(candidate)
|
||||
}
|
||||
this.stats.waitMs += Date.now() - waited
|
||||
this.dry = batch.length === 0
|
||||
if (batch.length === 0) {
|
||||
// Nothing to probe for a while: the connection closes (nntpIdleCloseMs), and opens again with work.
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (client && idleMs > 0 && Date.now() - usedAt >= idleMs) {
|
||||
client.close()
|
||||
client = null
|
||||
}
|
||||
this.stats.idleMs += 30_000
|
||||
await sleep(30_000)
|
||||
continue
|
||||
}
|
||||
try {
|
||||
client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
const fetched = Date.now()
|
||||
const bodies = await client.bodies(batch.map((c) => c.messageId))
|
||||
this.stats.fetchMs += Date.now() - fetched
|
||||
const inspected = Date.now()
|
||||
for (const [i, candidate] of batch.entries()) {
|
||||
const body = bodies[i]
|
||||
this.stats.articles++
|
||||
if (body === null) {
|
||||
// Taken down or expired: final.
|
||||
this.outcomes.push({ candidate, status: 'missing' })
|
||||
continue
|
||||
}
|
||||
this.stats.bytes += body.length
|
||||
try {
|
||||
this.outcomes.push({ candidate, status: 'done', inspection: inspect(body, candidate.kind) })
|
||||
} catch (error: any) {
|
||||
// Not yEnc, a broken one, or data a parser chokes on: a final answer, not a connection problem.
|
||||
if (!(error instanceof YencError)) logger.warn(`Probing ${candidate.messageId}: ${error.message}`)
|
||||
this.outcomes.push({ candidate, status: 'error' })
|
||||
}
|
||||
}
|
||||
this.stats.inspectMs += Date.now() - inspected
|
||||
usedAt = Date.now()
|
||||
} catch (error: any) {
|
||||
logger.warn(`Probing ${batch[0].messageId} and ${batch.length - 1} more: ${error.message}`)
|
||||
for (const candidate of batch) this.outcomes.push({ candidate, status: 'waiting' })
|
||||
client?.close()
|
||||
client = null
|
||||
await sleep(5000)
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
client?.close()
|
||||
this.running--
|
||||
}
|
||||
}
|
||||
|
||||
/** The queue's low-water mark and the level a claim fills it to (queueLevels). */
|
||||
private get levels() {
|
||||
return queueLevels(this.running, coordinatorConfig.probe.pipeline, coordinatorConfig.probe.batch)
|
||||
}
|
||||
|
||||
/** Claims to the queue's target; the claim after starts when it's below its low-water mark (queueLevels). */
|
||||
private get batch() {
|
||||
return this.levels.target
|
||||
}
|
||||
|
||||
private async next(): Promise<Candidate | undefined> {
|
||||
if (this.queue.length < this.levels.low) {
|
||||
this.refilling ??= this.claim().finally(() => (this.refilling = null))
|
||||
if (this.queue.length === 0) await this.refilling
|
||||
}
|
||||
return this.queue.shift()
|
||||
}
|
||||
|
||||
/**
|
||||
* Waiting candidates of settled releases, by kind, claimed so another round
|
||||
* doesn't take them: SLICES slices from random points of the queue. In id
|
||||
* order it went group by group, and some groups' articles take 0.6–3 s
|
||||
* where most take 20–60 ms (a.b.zogg 2016: 7 articles/s on 6 connections).
|
||||
*/
|
||||
private async claim() {
|
||||
const started = Date.now()
|
||||
try {
|
||||
await this.claimTiers()
|
||||
} finally {
|
||||
this.stats.claimMs += Date.now() - started
|
||||
}
|
||||
}
|
||||
|
||||
private async claimTiers() {
|
||||
// The queue empties, the lanes go dry and the share shrinks to its minimum until there's room again.
|
||||
if (await this.space.short()) return
|
||||
await this.claimCatalog()
|
||||
await this.claimPending()
|
||||
}
|
||||
|
||||
/** The pending ladder's share of a claim: at least pendingShare of the batch, and whatever the catalog's tiers left. */
|
||||
private async claimPending() {
|
||||
if (Date.now() - this.sweptAt > SWEEP_MS) {
|
||||
this.sweptAt = Date.now()
|
||||
this.stats.discarded += await PendingLadder.discard()
|
||||
}
|
||||
const want = Math.max(Math.floor(this.batch * coordinatorConfig.probe.pendingShare), this.batch - this.queue.length)
|
||||
const probes = await PendingLadder.claim(want)
|
||||
if (probes.length === 0) return
|
||||
const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
|
||||
probes.map((p) => p.messageId),
|
||||
PROBE_VERSION,
|
||||
])
|
||||
const known = new Map(cached.map((c: any) => [c.message_id, c]))
|
||||
for (const probe of probes) {
|
||||
const candidate: Candidate = { id: 0, releaseId: 0, kind: probe.kind, messageId: probe.messageId, pending: probe }
|
||||
const hit: any = known.get(probe.messageId)
|
||||
if (hit) {
|
||||
this.stats.cached++
|
||||
this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
|
||||
} else {
|
||||
this.queue.push(candidate)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async claimCatalog() {
|
||||
for (const { kind, unreadable } of TIERS) {
|
||||
const want = this.batch - this.queue.length
|
||||
if (want <= 0) return
|
||||
// The starting points chosen here: in the query Postgres drew random() once for all slices.
|
||||
// Over the claimable ones of this tier only: the rest of the queue isn't settled yet.
|
||||
// Per value of unreadable, each an end of the index: over `unreadable >= false` Postgres walked every waiting
|
||||
// entry of the kind, 2.5 minutes a claim at 28M archive candidates (LOG.md 2026-10-18 (74)).
|
||||
const values = unreadable ? [true] : [false, true]
|
||||
const ends = values
|
||||
.map(() => `SELECT min(id) AS lo, max(id) AS hi FROM probe_candidates WHERE status = 'waiting' AND ready AND kind = ? AND unreadable = ?`)
|
||||
.join(' UNION ALL ')
|
||||
const [{ lo, hi }] = (
|
||||
await db.rawQuery(
|
||||
`SELECT min(lo) AS lo, max(hi) AS hi FROM (${ends}) e`,
|
||||
values.flatMap((v) => [kind, v])
|
||||
)
|
||||
).rows
|
||||
if (lo === null) continue
|
||||
const starts = Array.from({ length: SLICES }, () => Number(lo) + Math.floor(Math.random() * (Number(hi) - Number(lo) + 1)))
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE probe_candidates SET status = 'taken' WHERE id IN (
|
||||
SELECT pc.id FROM unnest(?::bigint[]) AS o(start)
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT pc.id FROM probe_candidates pc
|
||||
WHERE pc.status = 'waiting' AND pc.ready AND pc.kind = ? AND pc.unreadable >= ? AND pc.id >= o.start
|
||||
ORDER BY pc.kind, pc.unreadable, pc.id LIMIT ? FOR UPDATE SKIP LOCKED) pc)
|
||||
RETURNING id, release_id, kind, message_id`,
|
||||
[starts, kind, unreadable, Math.ceil(want / SLICES)]
|
||||
)
|
||||
const claimed = rows.map((r: any) => ({ id: Number(r.id), releaseId: Number(r.release_id), kind: r.kind, messageId: r.message_id }))
|
||||
// Probed before (an earlier catalog's candidate of the same article): answered from probe_cache.
|
||||
const { rows: cached } = await db.rawQuery(`SELECT message_id, status, result FROM probe_cache WHERE message_id = ANY(?) AND version = ?`, [
|
||||
claimed.map((c: Candidate) => c.messageId),
|
||||
PROBE_VERSION,
|
||||
])
|
||||
const known = new Map(cached.map((c: any) => [c.message_id, c]))
|
||||
for (const candidate of claimed) {
|
||||
const hit: any = known.get(candidate.messageId)
|
||||
if (hit) {
|
||||
this.stats.cached++
|
||||
this.outcomes.push({ candidate, status: hit.status, inspection: hit.result ?? undefined, cached: true })
|
||||
} else {
|
||||
this.queue.push(candidate)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** The outcomes so far: candidates' status, inspections, and the releases' passworded and inner name. */
|
||||
private async flush() {
|
||||
const taken = this.outcomes.splice(0)
|
||||
if (taken.length === 0) return
|
||||
for (const o of taken) if (o.status !== 'waiting') this.stats[o.status]++
|
||||
// The ladder's outcomes apart from the catalog's: the next target, a name for the namer, or the end of the climb.
|
||||
const pending = taken.filter((o) => o.candidate.pending)
|
||||
const outcomes = taken.filter((o) => !o.candidate.pending)
|
||||
this.stats.pending += pending.filter((o) => o.status !== 'waiting').length
|
||||
await db.transaction(async (trx) => {
|
||||
if (outcomes.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET status = v.status FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, status text) WHERE pc.id = v.id`,
|
||||
[jsonb(outcomes.map((o) => ({ id: o.candidate.id, status: o.status })))]
|
||||
)
|
||||
}
|
||||
await PendingLadder.record(pending.map((o) => ({ probe: o.candidate.pending!, status: o.status, inspection: o.inspection })), trx)
|
||||
// Final answers read from the news server, kept by message-id for the next catalog (a reset).
|
||||
// One per message-id: the same article is a candidate of more than one release, and ON CONFLICT can't take a
|
||||
// row twice in one statement.
|
||||
const fresh = [...new Map(taken.filter((o) => !o.cached && o.status !== 'waiting').map((o) => [o.candidate.messageId, o])).values()]
|
||||
if (fresh.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO probe_cache (message_id, status, result, version)
|
||||
SELECT v.message_id, v.status, v.result, ? FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, status text, result jsonb)
|
||||
ON CONFLICT (message_id) DO UPDATE SET status = excluded.status, result = excluded.result, version = excluded.version, created_at = now()`,
|
||||
[PROBE_VERSION, jsonb(fresh.map((o) => ({ message_id: o.candidate.messageId, status: o.status, result: o.inspection ?? null })))]
|
||||
)
|
||||
}
|
||||
const done = outcomes.filter((o) => o.inspection)
|
||||
if (done.length === 0) return
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO release_inspections (release_id, candidate_id, kind, result)
|
||||
SELECT v.release_id, v.candidate_id, v.kind, v.result FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, candidate_id bigint, kind text, result jsonb)
|
||||
ON CONFLICT (release_id, candidate_id) DO UPDATE SET result = excluded.result, created_at = now()`,
|
||||
// A par2 set's file list goes to par2_files, not into every inspection.
|
||||
[jsonb(done.map((o) => ({ release_id: o.candidate.releaseId, candidate_id: o.candidate.id, kind: o.candidate.kind, result: { ...o.inspection, par2: undefined } })))]
|
||||
)
|
||||
await this.storeForMerge(trx, done)
|
||||
// A par2 index read: the release's completeness by its set, when it has no counter.
|
||||
await decidePar2Completeness(trx, [...new Set(done.filter((o) => o.inspection!.par2).map((o) => o.candidate.releaseId))])
|
||||
// Each touched release from all its inspections (earlier rounds may have stored others).
|
||||
const ids = [...new Set(done.map((o) => o.candidate.releaseId))]
|
||||
const { rows } = await trx.rawQuery(
|
||||
`SELECT i.release_id, i.kind, i.result, r.name FROM release_inspections i JOIN releases r ON r.id = i.release_id WHERE i.release_id = ANY(?)`,
|
||||
[ids]
|
||||
)
|
||||
const byRelease = new Map<number, { name: string; all: { kind: string; result: Inspection }[] }>()
|
||||
for (const r of rows) {
|
||||
const id = Number(r.release_id)
|
||||
const entry = byRelease.get(id) ?? { name: r.name as string, all: [] as { kind: string; result: Inspection }[] }
|
||||
entry.all.push({ kind: r.kind, result: r.result })
|
||||
byRelease.set(id, entry)
|
||||
}
|
||||
// A twin by content names a release its own inspections didn't (hash_twins.ts; LOG.md (130)).
|
||||
const twins = await twinNames(
|
||||
done.filter((o) => o.inspection!.hash16k && o.inspection!.size).map((o) => ({ hash16k: o.inspection!.hash16k!, size: o.inspection!.size })),
|
||||
trx
|
||||
)
|
||||
const twinOf = new Map<number, string>()
|
||||
for (const o of done) {
|
||||
const twin = o.inspection!.hash16k ? twins.get(twinKey(o.inspection!.hash16k, o.inspection!.size)) : undefined
|
||||
if (twin && twin !== byRelease.get(o.candidate.releaseId)?.name) twinOf.set(o.candidate.releaseId, twin)
|
||||
}
|
||||
const updates = [...byRelease].map(([id, { name, all }]) => ({
|
||||
id,
|
||||
passworded: all.some((i) => i.result.passworded === true),
|
||||
inner_name: innerName(all, name) ?? twinOf.get(id) ?? null,
|
||||
}))
|
||||
await trx.rawQuery(
|
||||
`UPDATE releases r SET passworded = v.passworded, inner_name = v.inner_name, inspected_at = now()
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, passworded boolean, inner_name text) WHERE r.id = v.id`,
|
||||
[jsonb(updates)]
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
/** What the par2 merge matches (app/services/par2_merger.ts): files' first-16-KiB hashes, par2 sets' file lists. */
|
||||
private async storeForMerge(trx: any, done: Outcome[]) {
|
||||
const hashed = done.filter((o) => o.inspection!.hash16k)
|
||||
if (hashed.length > 0) {
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO file_hashes (candidate_id, release_id, hash16k, size)
|
||||
SELECT v.candidate_id, v.release_id, v.hash16k, v.size FROM jsonb_to_recordset(?::jsonb) AS v(candidate_id bigint, release_id bigint, hash16k text, size bigint)
|
||||
ON CONFLICT (candidate_id) DO NOTHING`,
|
||||
[jsonb(hashed.map((o) => ({ candidate_id: o.candidate.id, release_id: o.candidate.releaseId, hash16k: o.inspection!.hash16k, size: o.inspection!.size })))]
|
||||
)
|
||||
}
|
||||
for (const o of done) {
|
||||
const set = o.inspection!.par2
|
||||
if (!set) continue
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO par2_files (set_id, file_id, hash16k, length, name)
|
||||
SELECT ?, v.file_id, v.hash16k, v.length, v.name FROM jsonb_to_recordset(?::jsonb) AS v(file_id text, hash16k text, length bigint, name text)
|
||||
ON CONFLICT DO NOTHING`,
|
||||
[set.setId, jsonb(set.files.map((f) => ({ file_id: f.fileId, hash16k: f.hash16k, length: f.length, name: f.name })))]
|
||||
)
|
||||
await trx.rawQuery(`INSERT INTO par2_sets (set_id, release_id, files) VALUES (?, ?, ?) ON CONFLICT DO NOTHING`, [set.setId, o.candidate.releaseId, set.files.length])
|
||||
}
|
||||
}
|
||||
|
||||
private report() {
|
||||
const seconds = (Date.now() - this.stats.since) / 1000
|
||||
if (seconds < REPORT_MS / 1000) return
|
||||
const { articles, bytes, done, missing, error } = this.stats
|
||||
logger.info(
|
||||
`Probed ${articles} articles in ${seconds.toFixed(0)} s: ${(articles / seconds).toFixed(1)}/s, ${((bytes * 8) / seconds / 1e6).toFixed(0)} Mbit/s; ` +
|
||||
`${done} inspected, ${missing} missing, ${error} not yEnc; ${this.stats.cached} from the probe cache; ` +
|
||||
`the pending ladder ${this.stats.pending} probed, ${this.stats.discarded} discarded by signature. ` +
|
||||
`Lanes: fetching ${(this.stats.fetchMs / 1000).toFixed(0)} s, waiting for candidates ${(this.stats.waitMs / 1000).toFixed(0)} s, ` +
|
||||
`idle ${(this.stats.idleMs / 1000).toFixed(0)} s, inspecting ${(this.stats.inspectMs / 1000).toFixed(1)} s; ` +
|
||||
`claims ${(this.stats.claimMs / 1000).toFixed(1)} s, flushes ${(this.stats.flushMs / 1000).toFixed(1)} s.`
|
||||
)
|
||||
this.stats = Prober.freshStats()
|
||||
}
|
||||
}
|
||||
@@ -1,290 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
|
||||
/** Raised when a rule changes: verdicts of an older version are judged again (a full round, right away). */
|
||||
export const JUDGE_VERSION = 1
|
||||
/** Between rounds of the changed releases (judge_dirty, noted by triggers; migration 4900). */
|
||||
const ROUND_MS = 2 * 60 * 1000
|
||||
/**
|
||||
* A full round (every release, each rule a scan of its table: 35–38 min) this often: for what no trigger notes. Time
|
||||
* (unusable a day after settling, a release final once the ranges beside it are read) and fewer releases with a file
|
||||
* after a merge (repeated).
|
||||
*/
|
||||
const FULL_MS = 24 * 60 * 60 * 1000
|
||||
/** Changed releases judged at a time. */
|
||||
const CHUNK = 50_000
|
||||
/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
|
||||
const REPEATED_RELEASES = 20
|
||||
|
||||
/**
|
||||
* A name that says it's a film, an episode or music: a resolution, a codec, a rip source, a media extension. No "?" in
|
||||
* these patterns: a raw query takes it for a binding ("h[ .]?26[45]" and "web[ .-]?dl" never matched; LOG.md
|
||||
* 2026-10-16 (64)), so {0,1}.
|
||||
*/
|
||||
const MEDIA =
|
||||
`(^|[^a-z0-9])(s[0-9]{1,2}e[0-9]{1,3}|[0-9]{3,4}p|x26[45]|h[ .]{0,1}26[45]|hevc|xvid|divx|bluray|bdrip|brrip|web[ .-]{0,1}dl|webrip|hdtv|dvdrip|dvdr|remux|mkv|mp4|avi|wmv|m4v|flac|mp3)([^a-z0-9]|$)`
|
||||
/** The posters of scraped celebrity pictures. */
|
||||
const SCRAPED_POSTERS = `celeb[-_ ]{0,1}movie|celebwow|celebfx|iluvcelebs|nude[-_ ]{0,1}celebs|celeb-sex-tapes`
|
||||
/** An executable's name. */
|
||||
const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
|
||||
|
||||
export interface Rule {
|
||||
name: string
|
||||
/** Why, in a few words (the log, the web interface). */
|
||||
says: string
|
||||
/** SELECT of the release ids the rule judges; scoped, only those in judge_scope. */
|
||||
sql: (scoped: boolean) => string
|
||||
}
|
||||
|
||||
/** A join to the releases in scope (an incremental round), on the given release id column. */
|
||||
const only = (scoped: boolean, column: string) => (scoped ? `JOIN judge_scope s ON s.release_id = ${column}` : '')
|
||||
|
||||
|
||||
|
||||
/**
|
||||
* The junk judge's rules, in order: a release takes the first that matches (REBUILD.md, junk). Each runs on what
|
||||
* the header pass, the prober, the merges and the spots know by now.
|
||||
*/
|
||||
export const RULES: Rule[] = [
|
||||
{
|
||||
name: 'dead',
|
||||
says: 'every probed first article is gone from the server (taken down or expired)',
|
||||
sql: (scoped) =>
|
||||
`SELECT c.release_id FROM probe_candidates c ${only(scoped, 'c.release_id')}
|
||||
WHERE c.status IN ('done', 'missing', 'error') GROUP BY 1 HAVING bool_and(c.status = 'missing')`,
|
||||
},
|
||||
{
|
||||
name: 'fake',
|
||||
says: 'named as a film, episode or music, but the content is a program',
|
||||
// An archive whose entries are all executables, under a media name (its own or its entries'); or a file that is
|
||||
// an executable by name or by its first bytes, under a media name.
|
||||
sql: (scoped) => `SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
|
||||
WHERE i.kind IN ('archive', '7z-tail') AND jsonb_array_length(i.result->'names') > 0
|
||||
AND NOT EXISTS (SELECT 1 FROM jsonb_array_elements_text(i.result->'names') n WHERE n !~* '${EXECUTABLE}')
|
||||
AND (r.name ~* '${MEDIA}' OR (i.result->>'names') ~* '${MEDIA}')
|
||||
UNION
|
||||
SELECT i.release_id FROM release_inspections i JOIN releases r ON r.id = i.release_id ${only(scoped, 'i.release_id')}
|
||||
WHERE i.kind = 'lone' AND ((i.result->>'yencName') ~* '${EXECUTABLE}' OR i.result->>'type' = 'exe')
|
||||
AND (r.name ~* '${MEDIA}' OR (i.result->>'yencName') ~* '${MEDIA}')`,
|
||||
},
|
||||
{
|
||||
name: 'repeated',
|
||||
says: `the same file content in ${REPEATED_RELEASES}+ releases (test files, spam)`,
|
||||
// Scoped: by the files of the releases in scope that are in REPEATED_RELEASES+ releases, counted once a chunk
|
||||
// (judge_tipped: ReleaseJudge.changed, which first adds the other releases of a file that just tipped over).
|
||||
sql: (scoped) =>
|
||||
scoped
|
||||
? `SELECT DISTINCT h.release_id FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id JOIN judge_tipped USING (hash16k, size)`
|
||||
: `SELECT DISTINCT h.release_id FROM file_hashes h
|
||||
JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
|
||||
},
|
||||
{
|
||||
name: 'scraped',
|
||||
says: 'celebrity pictures scraped from the web, reposted by one spam family',
|
||||
// "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
|
||||
// a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
|
||||
// titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
|
||||
// By the posters table's trigram index, each poster's releases then by releases_poster_newest_idx: 7 s, where the
|
||||
// regex on all of releases was 45 s, every round (LOG.md 2026-10-19 (80)).
|
||||
sql: (scoped) =>
|
||||
scoped
|
||||
? `SELECT r.id FROM releases r ${only(scoped, 'r.id')} JOIN posters p ON p.id = r.poster_id
|
||||
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`
|
||||
: `SELECT r.id FROM posters p JOIN releases r ON r.poster_id = p.id
|
||||
WHERE p.poster ~* '${SCRAPED_POSTERS}' AND r.source = ${SOURCES.subject} AND r.file_total IS NULL`,
|
||||
},
|
||||
{
|
||||
name: 'locked',
|
||||
says: 'passworded, and no spot, NZB or title gives the password',
|
||||
// releases_passworded_idx: a scan of all of releases was 30 s a round.
|
||||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')} WHERE r.passworded AND NOT EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id)`,
|
||||
},
|
||||
{
|
||||
name: 'unusable',
|
||||
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
|
||||
// Finished (settled_at): a provisional tail release's par2 or the rest of its files may still come.
|
||||
sql: (scoped) => `SELECT r.id FROM releases r ${only(scoped, 'r.id')}
|
||||
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
|
||||
AND r.settled_at IS NOT NULL
|
||||
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
|
||||
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
|
||||
},
|
||||
]
|
||||
|
||||
export interface JudgeRound {
|
||||
/** Releases judged: every one (a full round), or the changed ones and those a hash of theirs tipped over. */
|
||||
releases?: number
|
||||
/** Per rule: releases it judges now (in scope), verdicts it gained, verdicts it lost. */
|
||||
rules: Record<string, { matched: number; added: number; removed: number }>
|
||||
}
|
||||
|
||||
/**
|
||||
* The junk judge (REBUILD.md, junk; voyager 1's second filter, D42): after the header pass's rules (garbage
|
||||
* signatures, executables, …) a release is judged again once it's probed, by what its articles showed. Every few
|
||||
* minutes the releases whose facts changed (judge_dirty) are judged by every rule; a full round judges every release
|
||||
* daily. A release gets the verdict of the first rule in order that matches it; one that no longer matches loses its
|
||||
* verdict. Verdicts only hide (search leaves them out); nothing is dropped yet.
|
||||
*/
|
||||
export class ReleaseJudge {
|
||||
private stopped = false
|
||||
|
||||
stop() {
|
||||
this.stopped = true
|
||||
}
|
||||
|
||||
async run() {
|
||||
while (!this.stopped) {
|
||||
const started = Date.now()
|
||||
try {
|
||||
if (await this.fullDue()) {
|
||||
const round = await this.full()
|
||||
if (!this.stopped) logger.info(`Judged all releases: ${describe(round)}.`)
|
||||
} else {
|
||||
const round = await this.changed()
|
||||
if (round.releases! > 0) logger.info(`Judged ${round.releases!.toLocaleString('en')} changed releases: ${describe(round)}.`)
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Judge round failed: ${error.message}`)
|
||||
}
|
||||
for (let waited = Date.now() - started; waited < ROUND_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
|
||||
/** A full round is due: a day after the last, or the last was by another rule set. */
|
||||
async fullDue(): Promise<boolean> {
|
||||
const state = await db.from('judge_state').first()
|
||||
return !state || state.version !== JUDGE_VERSION || Date.now() - new Date(state.full_at).getTime() >= FULL_MS
|
||||
}
|
||||
|
||||
/**
|
||||
* Every release by every rule, the matches gathered in judge_full (each rule its own statement: no transaction of
|
||||
* half an hour), then applied at once. With dry, only counts. Stopped halfway, nothing is applied.
|
||||
*/
|
||||
async full(dry = false): Promise<JudgeRound> {
|
||||
const started = new Date()
|
||||
const round: JudgeRound = { rules: {} }
|
||||
// Verdicts of an older rule set are judged again from scratch.
|
||||
if (!dry) await db.from('release_verdicts').where('rule_version', '<>', JUDGE_VERSION).delete()
|
||||
await db.rawQuery(`TRUNCATE judge_full`)
|
||||
for (const rule of RULES) {
|
||||
if (this.stopped) return round
|
||||
const { rowCount } = await db.rawQuery(
|
||||
`INSERT INTO judge_full (release_id, rule) SELECT id, ? FROM (${rule.sql(false)}) AS x(id) ON CONFLICT DO NOTHING`,
|
||||
[rule.name]
|
||||
)
|
||||
round.rules[rule.name] = { matched: rowCount ?? 0, added: 0, removed: 0 }
|
||||
}
|
||||
if (dry || this.stopped) return round
|
||||
await db.rawQuery(`ANALYZE judge_full`)
|
||||
await db.transaction((trx) => this.apply(trx, 'judge_full', false, round))
|
||||
await db.rawQuery(
|
||||
`INSERT INTO judge_state (id, full_at, version) VALUES (1, ?, ?) ON CONFLICT (id) DO UPDATE SET full_at = excluded.full_at, version = excluded.version`,
|
||||
[started, JUDGE_VERSION]
|
||||
)
|
||||
return round
|
||||
}
|
||||
|
||||
/**
|
||||
* The releases noted in judge_dirty since, CHUNK at a time, each chunk in a transaction (a failed one stays noted).
|
||||
* A changed release can bring a file to REPEATED_RELEASES releases: the others with it are judged too (all but those
|
||||
* already repeated, or dead or fake, which come first). Only when it's the chunk's releases that bring it there: a
|
||||
* file in that many without them had its releases judged before.
|
||||
*/
|
||||
async changed(): Promise<JudgeRound> {
|
||||
const round: JudgeRound = { releases: 0, rules: {} }
|
||||
for (const rule of RULES) round.rules[rule.name] = { matched: 0, added: 0, removed: 0 }
|
||||
let more = true
|
||||
while (more && !this.stopped) {
|
||||
await db.transaction(async (trx) => {
|
||||
// Dropped first: inside an outer transaction (the tests) ON COMMIT DROP comes only at its end.
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judge_scope`)
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judged`)
|
||||
await trx.rawQuery(`CREATE TEMP TABLE judge_scope (release_id bigint PRIMARY KEY) ON COMMIT DROP`)
|
||||
const { rows } = await trx.rawQuery(
|
||||
`WITH taken AS (DELETE FROM judge_dirty WHERE ctid = ANY(ARRAY(SELECT ctid FROM judge_dirty LIMIT ${CHUNK})) RETURNING release_id),
|
||||
scoped AS (INSERT INTO judge_scope SELECT DISTINCT release_id FROM taken ON CONFLICT DO NOTHING RETURNING 1)
|
||||
SELECT (SELECT count(*) FROM taken) AS taken`
|
||||
)
|
||||
const taken = Number(rows[0].taken)
|
||||
more = taken === CHUNK
|
||||
if (taken === 0) return
|
||||
// The chunk's files in REPEATED_RELEASES+ releases, from file_hashes_hash_size_release_idx alone (migration 5000).
|
||||
await trx.rawQuery(`DROP TABLE IF EXISTS judge_tipped`)
|
||||
await trx.rawQuery(
|
||||
`CREATE TEMP TABLE judge_tipped (hash16k char(32), size bigint, just boolean NOT NULL, PRIMARY KEY (hash16k, size)) ON COMMIT DROP`
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO judge_tipped
|
||||
SELECT x.hash16k, x.size, c.outside < ${REPEATED_RELEASES}
|
||||
FROM (SELECT DISTINCT h.hash16k, h.size FROM file_hashes h JOIN judge_scope s ON s.release_id = h.release_id) x
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT count(DISTINCT f.release_id) AS total,
|
||||
count(DISTINCT f.release_id) FILTER (WHERE NOT EXISTS (SELECT 1 FROM judge_scope s WHERE s.release_id = f.release_id)) AS outside
|
||||
FROM file_hashes f WHERE f.hash16k = x.hash16k AND f.size = x.size) c
|
||||
WHERE c.total >= ${REPEATED_RELEASES}`
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`INSERT INTO judge_scope SELECT DISTINCT f.release_id FROM file_hashes f JOIN judge_tipped t USING (hash16k, size)
|
||||
WHERE t.just AND NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = f.release_id AND v.rule IN ('dead', 'fake', 'repeated'))
|
||||
ON CONFLICT DO NOTHING`
|
||||
)
|
||||
await trx.rawQuery(`ANALYZE judge_scope`)
|
||||
const { rows: scope } = await trx.rawQuery(`SELECT count(*) AS n FROM judge_scope`)
|
||||
round.releases! += Number(scope[0].n)
|
||||
await trx.rawQuery(`CREATE TEMP TABLE judged (release_id bigint PRIMARY KEY, rule varchar(16) NOT NULL) ON COMMIT DROP`)
|
||||
for (const rule of RULES) {
|
||||
const { rowCount } = await trx.rawQuery(
|
||||
`INSERT INTO judged (release_id, rule) SELECT id, ? FROM (${rule.sql(true)}) AS x(id) ON CONFLICT DO NOTHING`,
|
||||
[rule.name]
|
||||
)
|
||||
round.rules[rule.name].matched += rowCount ?? 0
|
||||
}
|
||||
await trx.rawQuery(`ANALYZE judged`)
|
||||
await this.apply(trx, 'judged', true, round)
|
||||
})
|
||||
}
|
||||
return round
|
||||
}
|
||||
|
||||
/**
|
||||
* The matches (judged: release_id, rule; the first rule's each) into release_verdicts: verdicts no rule gives any
|
||||
* more removed (scoped: of the releases in judge_scope only), changed ones written. Only the changes go through the
|
||||
* insert: each match through its conflict check took 60–150 s a rule (LOG.md 2026-10-19 (80)).
|
||||
*/
|
||||
private async apply(trx: any, judged: string, scoped: boolean, round: JudgeRound) {
|
||||
const tally = (rows: { rule: string; n: string }[], field: 'added' | 'removed') => {
|
||||
for (const { rule, n } of rows) {
|
||||
round.rules[rule] ??= { matched: 0, added: 0, removed: 0 }
|
||||
round.rules[rule][field] += Number(n)
|
||||
}
|
||||
}
|
||||
const { rows: removed } = await trx.rawQuery(
|
||||
`WITH gone AS (DELETE FROM release_verdicts v
|
||||
WHERE ${scoped ? 'v.release_id IN (SELECT release_id FROM judge_scope) AND' : ''}
|
||||
NOT EXISTS (SELECT 1 FROM ${judged} j WHERE j.release_id = v.release_id)
|
||||
RETURNING v.rule)
|
||||
SELECT rule, count(*) AS n FROM gone GROUP BY 1`
|
||||
)
|
||||
tally(removed, 'removed')
|
||||
const { rows: added } = await trx.rawQuery(
|
||||
`WITH written AS (INSERT INTO release_verdicts (release_id, rule, rule_version)
|
||||
SELECT j.release_id, j.rule, ? FROM ${judged} j LEFT JOIN release_verdicts v ON v.release_id = j.release_id
|
||||
WHERE v.rule IS DISTINCT FROM j.rule AND EXISTS (SELECT 1 FROM releases r WHERE r.id = j.release_id)
|
||||
ON CONFLICT (release_id) DO UPDATE SET rule = excluded.rule, rule_version = excluded.rule_version
|
||||
WHERE release_verdicts.rule <> excluded.rule
|
||||
RETURNING rule)
|
||||
SELECT rule, count(*) AS n FROM written GROUP BY 1`,
|
||||
[JUDGE_VERSION]
|
||||
)
|
||||
tally(added, 'added')
|
||||
}
|
||||
}
|
||||
|
||||
/** "dead 1,157,018 (+1,136 −712), …" for the log. */
|
||||
function describe(round: JudgeRound): string {
|
||||
return Object.entries(round.rules)
|
||||
.map(([name, r]) => `${name} ${r.matched.toLocaleString('en')} (+${r.added.toLocaleString('en')} −${r.removed.toLocaleString('en')})`)
|
||||
.join(', ')
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { DEFAULT_RULES, type Rules } from '#lib/rules/index'
|
||||
|
||||
type StoredRules = Omit<Rules, 'version' | 'priorities'>
|
||||
|
||||
/**
|
||||
* The rule set workers get with a lease (REBUILD.md, rules and recovery):
|
||||
* the newest version, plus the indexed groups' priorities (owner rule).
|
||||
* Version 1 (DEFAULT_RULES, no dropped signatures) is created when there is none.
|
||||
*/
|
||||
class RuleSets {
|
||||
async current(): Promise<Rules> {
|
||||
let row = await db.from('rule_sets').orderBy('version', 'desc').first()
|
||||
if (!row) {
|
||||
const { version: _version, ...rules } = DEFAULT_RULES
|
||||
await db.table('rule_sets').insert({ version: 1, rules: JSON.stringify({ ...rules, droppedSignatures: [] }), note: 'default' }).onConflict('version').ignore()
|
||||
row = await db.from('rule_sets').orderBy('version', 'desc').first()
|
||||
}
|
||||
const groups = await db.from('groups').where('active', true).select('name', 'priority')
|
||||
const { version: _v, ...defaults } = DEFAULT_RULES
|
||||
return {
|
||||
// A field newer than the stored set takes today's default.
|
||||
...defaults,
|
||||
...(row.rules as StoredRules),
|
||||
version: Number(row.version),
|
||||
priorities: Object.fromEntries(groups.map((g) => [g.name, Number(g.priority)])),
|
||||
}
|
||||
}
|
||||
|
||||
/** A new version: what changes is recorded on the ranges that run with it. */
|
||||
async publish(rules: StoredRules, note?: string): Promise<number> {
|
||||
const latest = await db.from('rule_sets').max('version as v').first()
|
||||
const version = Number(latest?.v ?? 0) + 1
|
||||
await db.table('rule_sets').insert({ version, rules: JSON.stringify(rules), note: note ?? null })
|
||||
return version
|
||||
}
|
||||
}
|
||||
|
||||
export default new RuleSets()
|
||||
@@ -1,349 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NzbService from '#services/nzb_service'
|
||||
import { SOURCES, sourceName } from '#lib/release/source'
|
||||
|
||||
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
||||
|
||||
/** Voyager 1's search parameters, on the catalog's columns (REBUILD.md, part 9). */
|
||||
export interface SearchParams {
|
||||
q?: string
|
||||
/** Also releases the junk judge hid (release_verdicts). */
|
||||
judged?: boolean
|
||||
/** Only releases with a file whose name is readable (not an obfuscated token), or an inner name from the prober. */
|
||||
readable?: boolean
|
||||
/** Only releases known to have all their files. */
|
||||
complete?: boolean
|
||||
/** Leave out releases known to miss files (Newznab). */
|
||||
notIncomplete?: boolean
|
||||
/** Leave out releases the prober found passworded (encrypted archive entries or headers). */
|
||||
unlocked?: boolean
|
||||
/** How the release was recognised: stem, subject or post. */
|
||||
source?: string
|
||||
poster?: string
|
||||
/** Exact category ids (already expanded from parents). */
|
||||
categories?: number[]
|
||||
maxAgeDays?: number
|
||||
/** TV: season and episode, matched as S01E02 or 1x02 in the name. */
|
||||
season?: number
|
||||
episode?: number
|
||||
/** Size bounds in bytes. */
|
||||
minSize?: number
|
||||
maxSize?: number
|
||||
sort?: SortOrder
|
||||
page?: number
|
||||
perPage?: number
|
||||
/** Overrides `page` (Newznab pages by offset). */
|
||||
offset?: number
|
||||
/** Count matches up to this many past the page; `total` is capped there. */
|
||||
countLimit?: number
|
||||
}
|
||||
|
||||
/** A release in search results. */
|
||||
export interface ReleaseSummary {
|
||||
id: number
|
||||
name: string
|
||||
source: string
|
||||
poster: string | null
|
||||
groups: string[]
|
||||
postedAt: Date | null
|
||||
fileTotal: number | null
|
||||
fileCount: number
|
||||
size: number
|
||||
readable: boolean
|
||||
complete: boolean | null
|
||||
/** The complete verdict is definite: the release is finished (voyager 3: the window closed on it; a tail release is provisional until then). */
|
||||
final: boolean
|
||||
category: number
|
||||
settled: boolean
|
||||
/** The prober found encrypted archive entries or headers; null: not probed (yet). */
|
||||
passworded: boolean | null
|
||||
/** A password is known (release_passwords): it's in the NZB. */
|
||||
passwordKnown: boolean
|
||||
/** What the prober found inside (an archive entry, a par2 set's file, a yEnc name), when it says more than the name. */
|
||||
innerName: string | null
|
||||
}
|
||||
|
||||
/** Posters a poster search looks in at most: a short fragment matches thousands of random ones (1,000 took 9.9 s, 200 0.2 s; LOG.md 2026-10-16 (70)). */
|
||||
const MAX_POSTERS = 200
|
||||
const SORTS: Record<SortOrder, string> = {
|
||||
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
|
||||
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
|
||||
largest: 'r.bytes DESC, r.id DESC',
|
||||
files: 'r.files DESC, r.id DESC',
|
||||
name: 'r.name ASC, r.id ASC',
|
||||
}
|
||||
|
||||
/** A broad search needn't count every match (each counted row is a heap read). */
|
||||
const COUNT_LIMIT = 10_000
|
||||
const STATS_TTL_MS = 5 * 60 * 1000
|
||||
/** A refresh that takes longer is given up (the next request tries again). */
|
||||
const STATS_TIMEOUT_MS = 2 * 60 * 1000
|
||||
|
||||
/** ILIKE pattern of one word (the words are split on dots, dashes, underscores and spaces). */
|
||||
function contains(word: string): string {
|
||||
return `%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`
|
||||
}
|
||||
|
||||
/**
|
||||
* A search word's condition on release_texts: anywhere in a text (ILIKE), or with a leading "^" at a word's start, the
|
||||
* text's or after a character that's no letter or digit ("^zed" finds "Zed.Dead", "The_Zed", not "Facialized"). Both
|
||||
* use the trigram index.
|
||||
*/
|
||||
function wordMatch(word: string): { sql: string; binding: string } {
|
||||
if (word.startsWith('^') && word.length > 1) {
|
||||
const rest = word.slice(1).replace(/[\\^$.|?*+()[\]{}]/g, (c) => `\\${c}`)
|
||||
return { sql: 't.text ~* ?', binding: `(^|[^[:alnum:]])${rest}` }
|
||||
}
|
||||
return { sql: 't.text ILIKE ?', binding: contains(word) }
|
||||
}
|
||||
|
||||
/** Searching and browsing the catalog, for the web interface and Newznab. */
|
||||
class SearchService {
|
||||
async search(params: SearchParams) {
|
||||
const where: string[] = ['r.files > 0']
|
||||
// Judged junk (app/services/release_judge.ts) is left out unless asked for.
|
||||
if (!params.judged) where.push('NOT EXISTS (SELECT 1 FROM release_verdicts v WHERE v.release_id = r.id)')
|
||||
const bindings: unknown[] = []
|
||||
// Every word in one of the release's texts (its name, the names the prober found inside; release_texts, by its
|
||||
// trigram index, which yields ids: never an OR across the two tables); "The.Block.AU" and "the block au" alike;
|
||||
// "^zed" only at a word's start.
|
||||
for (const word of (params.q ?? '').split(/[\s._-]+/).filter(Boolean).slice(0, 8)) {
|
||||
const match = wordMatch(word)
|
||||
where.push(`r.id IN (SELECT t.release_id FROM release_texts t WHERE ${match.sql})`)
|
||||
bindings.push(match.binding)
|
||||
}
|
||||
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
|
||||
if (params.complete) where.push('r.complete')
|
||||
// Incomplete only when final: a provisional tail release may yet get its missing files.
|
||||
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR r.settled_at IS NULL)')
|
||||
if (params.unlocked) where.push('r.passworded IS NOT TRUE')
|
||||
if (params.source) {
|
||||
const code = (SOURCES as Record<string, number>)[params.source]
|
||||
if (code === undefined) where.push('false')
|
||||
else {
|
||||
where.push('r.source = ?')
|
||||
bindings.push(code)
|
||||
}
|
||||
}
|
||||
// A poster search: the matching posters first (posters, by trigram), each one's releases then by
|
||||
// releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
|
||||
// poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
|
||||
let posters: number[] | null = null
|
||||
if (params.poster) {
|
||||
const { rows: matched } = await db.rawQuery(`SELECT id FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
|
||||
posters = matched.map((m: { id: string }) => Number(m.id))
|
||||
}
|
||||
if (params.categories?.length) {
|
||||
where.push('r.category = ANY(?)')
|
||||
bindings.push(params.categories)
|
||||
}
|
||||
if (params.maxAgeDays) {
|
||||
where.push('r.posted_at >= now() - make_interval(days => ?)')
|
||||
bindings.push(params.maxAgeDays)
|
||||
}
|
||||
if (params.minSize !== undefined) {
|
||||
where.push('r.bytes >= ?')
|
||||
bindings.push(params.minSize)
|
||||
}
|
||||
if (params.maxSize !== undefined) {
|
||||
where.push('r.bytes <= ?')
|
||||
bindings.push(params.maxSize)
|
||||
}
|
||||
if (params.season !== undefined) {
|
||||
const s = params.season
|
||||
const e = params.episode
|
||||
where.push('r.name ~* ?')
|
||||
bindings.push(e !== undefined ? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)` : `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`)
|
||||
}
|
||||
|
||||
const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
|
||||
const page = Math.max(params.page ?? 1, 1)
|
||||
const offset = params.offset ?? (page - 1) * perPage
|
||||
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
||||
const whereSql = where.join(' AND ')
|
||||
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
||||
if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
|
||||
// Per poster found: its releases matching everything else (releases_poster_newest_idx), the first offset + perPage
|
||||
// in the order asked; then merged.
|
||||
const source =
|
||||
posters === null
|
||||
? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
|
||||
: {
|
||||
from: `unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
|
||||
SELECT r.* FROM releases r WHERE r.poster_id = p.id AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
|
||||
where: 'true',
|
||||
bindings: [posters] as unknown[],
|
||||
}
|
||||
const counting =
|
||||
posters === null
|
||||
? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
|
||||
: db.rawQuery(
|
||||
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::int[]) AS p(id) CROSS JOIN LATERAL (
|
||||
SELECT 1 FROM releases r WHERE r.poster_id = p.id AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
|
||||
[posters, ...bindings]
|
||||
)
|
||||
const [{ rows }, { rows: counted }] = await Promise.all([
|
||||
db.rawQuery(
|
||||
`SELECT r.id, r.name, r.source, po.poster, gs.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
||||
EXISTS (SELECT 1 FROM release_passwords p WHERE p.release_id = r.id) AS password_known,
|
||||
r.settled_at IS NOT NULL AS final
|
||||
FROM ${source.from} LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id
|
||||
WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
[...source.bindings, ...bindings, perPage, offset]
|
||||
),
|
||||
counting,
|
||||
])
|
||||
return {
|
||||
page,
|
||||
perPage,
|
||||
offset,
|
||||
total: Math.min(Number(counted[0].total), countLimit),
|
||||
/** More than `total` match (the count stops at `countLimit`). */
|
||||
totalCapped: Number(counted[0].total) > countLimit,
|
||||
releases: (rows as any[]).map((r): ReleaseSummary => ({
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
source: sourceName(r.source),
|
||||
poster: r.poster ?? null,
|
||||
groups: r.groups ?? [],
|
||||
postedAt: r.posted_at,
|
||||
fileTotal: r.file_total,
|
||||
fileCount: Number(r.files),
|
||||
size: Number(r.bytes),
|
||||
readable: r.readable,
|
||||
complete: r.complete,
|
||||
final: r.final,
|
||||
category: r.category,
|
||||
settled: r.settled_at !== null,
|
||||
passworded: r.passworded,
|
||||
passwordKnown: r.password_known,
|
||||
innerName: r.inner_name,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/** A release with its files, read from its chunks. */
|
||||
async release(id: number) {
|
||||
const {
|
||||
rows: [r],
|
||||
} = await db.rawQuery(
|
||||
`SELECT r.*, po.poster, gs.groups FROM releases r LEFT JOIN posters po ON po.id = r.poster_id LEFT JOIN group_sets gs ON gs.id = r.group_set_id WHERE r.id = ?`,
|
||||
[id]
|
||||
)
|
||||
if (!r) return null
|
||||
const [files, { rows: inspections }, passwords] = await Promise.all([
|
||||
NzbService.files(id),
|
||||
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
|
||||
NzbService.passwords(id),
|
||||
])
|
||||
return {
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
source: sourceName(r.source),
|
||||
poster: r.poster ?? null,
|
||||
groups: r.groups ?? [],
|
||||
postedAt: r.posted_at,
|
||||
fileTotal: r.file_total,
|
||||
fileCount: Number(r.files),
|
||||
size: Number(r.bytes),
|
||||
segments: Number(r.segments),
|
||||
readable: r.readable,
|
||||
complete: r.complete,
|
||||
final: r.settled_at !== null,
|
||||
hasPar2: r.has_par2,
|
||||
category: r.category,
|
||||
firstSeenAt: r.first_seen_at,
|
||||
settledAt: r.settled_at,
|
||||
passworded: r.passworded,
|
||||
/** Known passwords (spots, their NZBs), the most trusted first; the first is in the NZB. */
|
||||
passwords,
|
||||
innerName: r.inner_name,
|
||||
inspectedAt: r.inspected_at,
|
||||
/** What the prober read from the release's articles (lib/probe Inspection), by probe kind. */
|
||||
inspections: (inspections as any[]).map((i) => ({ kind: i.kind, ...i.result, at: i.created_at })),
|
||||
files: files.map((f) => ({
|
||||
name: f.name,
|
||||
subject: f.subject,
|
||||
poster: f.poster,
|
||||
postedAt: f.postedAt === null ? null : new Date(f.postedAt),
|
||||
size: f.segments.reduce((n, [, , bytes]) => n + bytes, 0),
|
||||
parts: f.segments.length,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/** Totals for the web interface, computed at most once a minute (callers in between share them). */
|
||||
async stats() {
|
||||
if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
|
||||
this.statsRefresh ??= Promise.race([
|
||||
this.computeStats(),
|
||||
new Promise<never>((_, reject) => setTimeout(() => reject(new Error('timed out')), STATS_TIMEOUT_MS).unref()),
|
||||
])
|
||||
.then((value) => {
|
||||
this.statsCache = { at: Date.now(), value }
|
||||
})
|
||||
.catch((error) => {
|
||||
logger.warn(`Stats: ${error.message}`)
|
||||
if (!this.statsCache) throw error
|
||||
})
|
||||
.finally(() => {
|
||||
this.statsRefresh = undefined
|
||||
})
|
||||
if (!this.statsCache) await this.statsRefresh
|
||||
}
|
||||
return this.statsCache!.value
|
||||
}
|
||||
|
||||
private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
|
||||
private statsRefresh?: Promise<void>
|
||||
|
||||
/**
|
||||
* Estimates (a catalog of 100M+ rows can't be counted on every visit):
|
||||
* the row count from Postgres's statistics, below a million rows exact;
|
||||
* readable share, files and size from a sample of 5,000 rows; ranges
|
||||
* counted exactly (a few hundred thousand rows).
|
||||
*/
|
||||
private async computeStats() {
|
||||
const { rows } = await db.rawQuery(`SELECT reltuples::bigint AS n FROM pg_class WHERE relname = 'releases'`)
|
||||
const estimate = Number(rows[0]?.n ?? 0)
|
||||
// A fixed-size sample: a percentage of a big catalog is many random reads on the HDD.
|
||||
const sample = estimate < 1_000_000 ? '' : 'TABLESAMPLE SYSTEM_ROWS (5000)'
|
||||
const { rows: s } = await db.rawQuery(
|
||||
`SELECT count(*) AS sampled, count(*) FILTER (WHERE readable) AS readable, coalesce(sum(files), 0) AS files, coalesce(sum(bytes), 0) AS bytes
|
||||
FROM releases ${sample} WHERE files > 0`
|
||||
)
|
||||
const sampled = Number(s[0].sampled)
|
||||
const releases = sample ? Math.max(estimate, 0) : sampled
|
||||
const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
|
||||
// History by articles; the pass: the history ranges before those deferred behind them (priority
|
||||
// 2000 apart; e.g. groups added later that go first).
|
||||
const { rows: ranges } = await db.rawQuery(
|
||||
`WITH h AS (SELECT status, priority, end_id - start_id + 1 AS n FROM ranges WHERE priority >= 0 AND status <> 'ancient'),
|
||||
pass AS (SELECT (floor(coalesce(min(priority) FILTER (WHERE status <> 'done'), 0) / 2000) + 1) * 2000 AS below FROM h)
|
||||
SELECT (SELECT count(*) FILTER (WHERE status = 'done') FROM ranges) AS done, (SELECT count(*) FROM ranges WHERE status <> 'ancient') AS total,
|
||||
coalesce(sum(n) FILTER (WHERE status = 'done'), 0) AS history_done, coalesce(sum(n), 0) AS history,
|
||||
coalesce(sum(n) FILTER (WHERE status = 'done' AND priority < (SELECT below FROM pass)), 0) AS pass_done,
|
||||
coalesce(sum(n) FILTER (WHERE priority < (SELECT below FROM pass)), 0) AS pass,
|
||||
(SELECT max(done_at) FROM ranges) AS last_done_at,
|
||||
(SELECT max(done_at) FROM ranges WHERE priority < 0) AS tail_done_at
|
||||
FROM h`
|
||||
)
|
||||
const g = ranges[0]
|
||||
return {
|
||||
approximate: sample !== '',
|
||||
releases,
|
||||
readableReleases: scale(s[0].readable),
|
||||
files: scale(s[0].files),
|
||||
size: scale(s[0].bytes),
|
||||
ranges: { done: Number(g.done), total: Number(g.total) },
|
||||
/** History and the current pass, in articles. */
|
||||
history: { done: Number(g.history_done), total: Number(g.history) },
|
||||
pass: { done: Number(g.pass_done), total: Number(g.pass) },
|
||||
lastIndexedAt: g.last_done_at,
|
||||
tailIndexedAt: g.tail_done_at,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new SearchService()
|
||||
@@ -1,36 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
/** Hours a complete provisional release stays open for its window's continuation before it is settled here. */
|
||||
export const SETTLE_COMPLETE_HOURS = 6
|
||||
|
||||
/**
|
||||
* Provisional releases (REBUILD.md, "Voyager 3: window indexing", 2): a release the window hadn't closed on at a
|
||||
* lease's end is written with `settled_at` null, for the window's continuation to finish. A continuation may
|
||||
* never come (the range-lease path of the first run left 109k such rows, LOG.md (118); a group's next lease is
|
||||
* hours away), and the catalog shows the release as still collecting. A complete one — every file its counter
|
||||
* announced found — can't gain a counted file any more, so once it has been quiet for SETTLE_COMPLETE_HOURS it is
|
||||
* settled here, as the finish would: `settled_at` set (the judge's trigger fires on it), its waiting probe
|
||||
* candidates made claimable. Incomplete ones keep waiting: the continuation adds their files.
|
||||
*/
|
||||
class Settler {
|
||||
async settleComplete(quietHours = SETTLE_COMPLETE_HOURS): Promise<number> {
|
||||
return db.transaction(async (trx) => {
|
||||
const { rows } = await trx.rawQuery(
|
||||
`UPDATE releases SET settled_at = now()
|
||||
WHERE settled_at IS NULL AND complete
|
||||
AND first_seen_at < now() - make_interval(hours => ?) AND last_posted_at < now() - make_interval(hours => ?)
|
||||
RETURNING id`,
|
||||
[quietHours, quietHours]
|
||||
)
|
||||
if (rows.length > 0)
|
||||
await trx.rawQuery(
|
||||
`UPDATE probe_candidates pc SET ready = true, unreadable = NOT r.readable FROM releases r
|
||||
WHERE pc.release_id = ANY(?::bigint[]) AND r.id = pc.release_id AND pc.status = 'waiting' AND NOT pc.ready`,
|
||||
[rows.map((r: { id: string }) => Number(r.id))]
|
||||
)
|
||||
return rows.length
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default new Settler()
|
||||
@@ -1,22 +0,0 @@
|
||||
/**
|
||||
* 7z keeps its listing at the end of the last volume, so a split 7z gets a `7z-tail` probe candidate: the last
|
||||
* segment of its highest volume (lib/edge/range_indexer.ts). RAR lists its files in every volume's header, and
|
||||
* its first volume's start (one `archive` candidate) is enough. The live tail reads an upload in pieces while
|
||||
* it's posted, and each piece queues a tail for the last volume it saw so far (TsIgNBewuFbuwWy: 7, for
|
||||
* .002–.008; LOG.md 2026-10-23 (94)); merges bring their members' tails together. Of the given releases'
|
||||
* waiting tails, those below the release's highest volume go. Those of the highest volume stay, all of them:
|
||||
* a piece that saw only part of it queued a segment that isn't its last.
|
||||
*/
|
||||
export async function pruneSevenZipTails(trx: any, releaseIds: (number | string)[]): Promise<number> {
|
||||
if (releaseIds.length === 0) return 0
|
||||
const result = await trx.rawQuery(
|
||||
`DELETE FROM probe_candidates pc USING (
|
||||
SELECT id, status, volume, max(volume) OVER (PARTITION BY release_id) AS highest
|
||||
FROM (SELECT id, release_id, status, substring(file from '\\.7z\\.(\\d{1,4})$')::int AS volume
|
||||
FROM probe_candidates WHERE release_id = ANY(?) AND kind = '7z-tail') t
|
||||
) s
|
||||
WHERE pc.id = s.id AND s.status = 'waiting' AND s.volume < s.highest`,
|
||||
[releaseIds.map(String)]
|
||||
)
|
||||
return result.rowCount ?? 0
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
|
||||
|
||||
/** Candidates looked at per round. */
|
||||
const BATCH = 2000
|
||||
/** An upload's sidecar is posted with it: its release is looked for this far from it. */
|
||||
const WINDOW_MINUTES = 10
|
||||
/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
|
||||
export const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
|
||||
|
||||
/** The words of a name worth comparing: 4+ letters or digits. */
|
||||
function words(text: string): Set<string> {
|
||||
return new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4))
|
||||
}
|
||||
|
||||
export interface Sibling {
|
||||
id: number
|
||||
name: string
|
||||
files: number
|
||||
file_total: number | null
|
||||
}
|
||||
|
||||
/**
|
||||
* Which of the nearby releases a sidecar belongs to: the one sharing a word of 4+ letters with the sidecar's subject
|
||||
* ("Murdoch Mysteries S8D1 [00/80] - "Murdoch Mysteries S8D1.nzb"" beside "Murdoch Mysteries S8D1"), else the one
|
||||
* under the same counter total ("(ALADIN) [01/34] - "bud.txt"" beside the poster's 34-file upload); none when that
|
||||
* isn't one release.
|
||||
*/
|
||||
export function pickSibling(files: ChunkFile[], fileTotal: number | null, siblings: Sibling[]): Sibling | null {
|
||||
const said = words(files.map((f) => f.subject).join(' '))
|
||||
const named = siblings.filter((s) => [...words(s.name)].some((w) => said.has(w)))
|
||||
if (named.length === 1) return named[0]
|
||||
if (named.length > 1) return null
|
||||
const counted = fileTotal ? siblings.filter((s) => s.file_total === fileTotal) : []
|
||||
return counted.length === 1 ? counted[0] : null
|
||||
}
|
||||
|
||||
export interface FoldRound {
|
||||
looked: number
|
||||
sidecars: number
|
||||
folded: number
|
||||
}
|
||||
|
||||
/**
|
||||
* The sidecar fold (REBUILD.md, junk): an .nfo, .sfv or .nzb that came apart from its upload (another name, another
|
||||
* counter: "[00/80] … .nzb" beside "[01/94] … .part01.rar") was a release of its own. The settler queues small
|
||||
* settled releases (fold_candidates); a candidate whose files are all sidecars joins the poster's bigger release in
|
||||
* the same group posted within WINDOW_MINUTES, the one pickSibling chooses, through the merger's merge (via
|
||||
* 'sidecar'). The rest leave the queue untouched.
|
||||
*/
|
||||
export class SidecarFolder {
|
||||
constructor(private merge: (trx: any, found: Map<string, any>, key: string, via: 'sidecar') => Promise<number>) {}
|
||||
|
||||
async round(): Promise<FoldRound> {
|
||||
const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
|
||||
// A candidate whose release went (merged, deleted) leaves with its batch: a separate scan for those over the whole
|
||||
// queue took 5+ min a round at 13M (LOG.md 2026-10-17 (72)).
|
||||
const { rows: queued } = await db.rawQuery(
|
||||
`SELECT f.release_id AS queued_id, r.id, r.group_id, r.poster_id, r.posted_at, r.files, r.file_total, r.readable
|
||||
FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
|
||||
[BATCH]
|
||||
)
|
||||
const done: number[] = queued.map((r: { queued_id: string }) => Number(r.queued_id))
|
||||
for (const c of queued) {
|
||||
if (c.id === null) continue
|
||||
round.looked++
|
||||
const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
|
||||
const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
|
||||
if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
|
||||
round.sidecars++
|
||||
if (!c.posted_at) continue
|
||||
const { rows: siblings } = await db.rawQuery(
|
||||
`SELECT r.id, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
|
||||
WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
|
||||
AND r.group_id = ? AND r.poster_id = ? AND r.id <> ? AND r.files > ?`,
|
||||
[c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster_id, c.id, c.files]
|
||||
)
|
||||
const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
|
||||
if (!sibling) continue
|
||||
const full = siblings.find((s: any) => Number(s.id) === sibling.id)
|
||||
const merged = await db.transaction((trx) =>
|
||||
this.merge(trx, new Map([[String(c.id), c], [String(full.id), full]]), `sidecar:${full.id}`, 'sidecar')
|
||||
)
|
||||
if (merged > 0) round.folded++
|
||||
}
|
||||
if (done.length > 0) await db.from('fold_candidates').whereIn('release_id', done).delete()
|
||||
return round
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
import { mkdir, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
|
||||
export interface ChunkSpan {
|
||||
from: number
|
||||
to: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Prefetched XOVER chunks on disk (REBUILD.md, prefetch): `<dir>/<range id>/<from>-<to>.gz`,
|
||||
* each as the news server sent it. Chunks follow a fixed grid from the
|
||||
* range's start, so a resume point (always at a batch boundary) is a chunk's start.
|
||||
*/
|
||||
class Spool {
|
||||
get dir() {
|
||||
return coordinatorConfig.spool.dir
|
||||
}
|
||||
|
||||
/** The chunks of a range from `from` (its resume point) to its end. */
|
||||
chunksOf(range: { start: number; end: number }, from = range.start): ChunkSpan[] {
|
||||
const size = coordinatorConfig.chunkArticles
|
||||
const spans: ChunkSpan[] = []
|
||||
for (let at = range.start; at <= range.end; at += size) {
|
||||
const span = { from: at, to: Math.min(at + size - 1, range.end) }
|
||||
if (span.to >= from) spans.push(span)
|
||||
}
|
||||
return spans
|
||||
}
|
||||
|
||||
private path(rangeId: number, span: ChunkSpan) {
|
||||
return join(this.dir, String(rangeId), `${span.from}-${span.to}.gz`)
|
||||
}
|
||||
|
||||
async has(rangeId: number, span: ChunkSpan) {
|
||||
try {
|
||||
await stat(this.path(rangeId, span))
|
||||
return true
|
||||
} catch {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
async write(rangeId: number, span: ChunkSpan, data: Buffer) {
|
||||
const file = this.path(rangeId, span)
|
||||
await mkdir(join(this.dir, String(rangeId)), { recursive: true })
|
||||
// Written under another name first: a reader never sees half a chunk.
|
||||
await writeFile(`${file}.part`, data)
|
||||
await rename(`${file}.part`, file)
|
||||
}
|
||||
|
||||
read(rangeId: number, span: ChunkSpan) {
|
||||
return readFile(this.path(rangeId, span))
|
||||
}
|
||||
|
||||
/** One chunk, e.g. one that doesn't inflate (a false end of reply; see XoverClient.compressedRaw). */
|
||||
async removeChunk(rangeId: number, span: ChunkSpan) {
|
||||
await rm(this.path(rangeId, span), { force: true })
|
||||
}
|
||||
|
||||
async remove(rangeId: number) {
|
||||
await rm(join(this.dir, String(rangeId)), { recursive: true, force: true })
|
||||
}
|
||||
|
||||
/** Bytes in the spool. */
|
||||
async size(): Promise<number> {
|
||||
let total = 0
|
||||
let ranges: string[] = []
|
||||
try {
|
||||
ranges = await readdir(this.dir)
|
||||
} catch {
|
||||
return 0
|
||||
}
|
||||
for (const range of ranges) {
|
||||
for (const file of await readdir(join(this.dir, range)).catch(() => [])) {
|
||||
total += (await stat(join(this.dir, range, file)).catch(() => ({ size: 0 }))).size
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
}
|
||||
|
||||
export default new Spool()
|
||||
@@ -1,381 +0,0 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import { gzipSync } from 'node:zlib'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { decodeSpotBinary } from '#lib/spotnet/index'
|
||||
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
||||
import { parseSubject } from '#lib/headers/index'
|
||||
import { releaseIdentity } from '#lib/release/index'
|
||||
import { identityKey, releaseKey } from '#lib/release/keys'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
import CatalogRefs from '#services/catalog_refs'
|
||||
|
||||
/** Spots claimed per round, newest first. */
|
||||
const BATCH = 20
|
||||
/** Between rounds when there was nothing to do. */
|
||||
const IDLE_MS = 60_000
|
||||
/** Unmatched spots are matched again this often (the header pass may have built their releases since). */
|
||||
const REMATCH_MS = 60 * 60 * 1000
|
||||
/**
|
||||
* Unmatched spots matched again per hour: after a reset every spot is (16.5k at 500 an hour took 33 h). Never tried
|
||||
* first, then the newest: the oldest inspection first tried a 3-month backfill before today's spots, the ones whose
|
||||
* releases the tail has (2 of 16.7k matched 3 h after reset 24).
|
||||
*/
|
||||
const REMATCH_BATCH = 5000
|
||||
/** Unmatched spots are matched again for this long after their inspection. */
|
||||
const REMATCH_DAYS = 30
|
||||
/** A file's release posted further apart than this from the NZB's date is another upload of the same name. */
|
||||
const MAX_SPREAD_MS = 48 * 3600 * 1000
|
||||
|
||||
export interface InspectRound {
|
||||
inspected: number
|
||||
matched: number
|
||||
standalone: number
|
||||
outcomes: Record<string, number>
|
||||
}
|
||||
|
||||
interface SpotFile {
|
||||
idx: number
|
||||
name: string
|
||||
subject: string
|
||||
poster: string
|
||||
groups: string[]
|
||||
postedAt: Date | null
|
||||
bytes: number
|
||||
segments: number
|
||||
}
|
||||
|
||||
/**
|
||||
* A placeholder NZB (Dummy.rar + Dummy.nzb, shared by many spots): under 1% of the size the spot announces, or under
|
||||
* 16 KB in all (text-only spots). Its files aren't listed (voyager 1's isDummyNzb).
|
||||
*/
|
||||
export function isDummyNzb(nzb: ParsedNzb, spotSize: number): boolean {
|
||||
const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
|
||||
return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
|
||||
}
|
||||
|
||||
/** A file of an NZB as the header pass names it: the subject's file name, by the same parser. */
|
||||
export function nzbFileName(subject: string): { name: string; base: string } {
|
||||
const parsed = parseSubject(subject) ?? parseSubject(`${subject} (1/1)`)
|
||||
return parsed ? { name: parsed.name, base: parsed.base } : { name: subject, base: subject }
|
||||
}
|
||||
|
||||
/**
|
||||
* The spot inspector (REBUILD.md, spots): looks inside each listed spot, on
|
||||
* one NNTP connection of its own. It reads the spot's compressed NZB (its
|
||||
* articles in alt.binaries.ftd), keeps the list of files in `spot_files`,
|
||||
* and matches every file to the release the header pass built for it: the
|
||||
* release identity of the file's subject and poster (lib/release, as the
|
||||
* edge workers compute it), in one of the file's groups, posted within 48 h.
|
||||
* Each matched release is linked in `release_spots`, and gets the spot's
|
||||
* password in `release_passwords` (the NZB's own `<meta type="password">`
|
||||
* first, else what the spot's title or text names) and `passworded`. A spot
|
||||
* whose files match nothing is matched again every hour, for 30 days; once
|
||||
* SPOT_STANDALONE_AFTER_HOURS have passed since its inspection it becomes a
|
||||
* release of its own (source `spot`) with its NZB, until the header pass
|
||||
* builds the releases after all (a spot older than what the catalog has
|
||||
* indexed): then it moves to those, and its own release goes.
|
||||
*/
|
||||
export class SpotInspector {
|
||||
private stopped = false
|
||||
private client?: XoverClient
|
||||
private groupIds = new Map<string, number | null>()
|
||||
private rematchedAt = 0
|
||||
|
||||
stop() {
|
||||
this.stopped = true
|
||||
this.client?.close()
|
||||
}
|
||||
|
||||
async run() {
|
||||
while (!this.stopped) {
|
||||
let round: InspectRound | null = null
|
||||
try {
|
||||
round = await this.round()
|
||||
if (round.inspected > 0 || round.standalone > 0) {
|
||||
const outcomes = Object.entries(round.outcomes).map(([k, n]) => `${n} ${k}`).join(', ')
|
||||
logger.info(`Spots inspected: ${round.inspected} (${outcomes}); ${round.matched} matched releases, ${round.standalone} of their own.`)
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Spot inspection failed: ${error.message}`)
|
||||
this.client?.close()
|
||||
this.client = undefined
|
||||
}
|
||||
if (!round || round.inspected === 0) {
|
||||
// Nothing to inspect for a while: closed (nntpIdleCloseMs), opened again with work.
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (idleMs > 0 && IDLE_MS >= idleMs) {
|
||||
this.client?.close()
|
||||
this.client = undefined
|
||||
}
|
||||
for (let waited = 0; waited < IDLE_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async connect(): Promise<XoverClient> {
|
||||
this.client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
return this.client
|
||||
}
|
||||
|
||||
async round(): Promise<InspectRound> {
|
||||
const round: InspectRound = { inspected: 0, matched: 0, standalone: 0, outcomes: {} }
|
||||
const spots = await db.from('spots').where('status', 'listed').orderBy('posted_at', 'desc').limit(BATCH)
|
||||
for (const spot of spots) {
|
||||
if (this.stopped) break
|
||||
const outcome = await this.inspect(spot)
|
||||
round.inspected++
|
||||
round.outcomes[outcome.status] = (round.outcomes[outcome.status] ?? 0) + 1
|
||||
round.matched += outcome.matched
|
||||
}
|
||||
if (Date.now() - this.rematchedAt >= REMATCH_MS && !this.stopped) {
|
||||
this.rematchedAt = Date.now()
|
||||
const again = await this.rematch()
|
||||
round.matched += again.matched
|
||||
round.standalone += again.standalone
|
||||
}
|
||||
return round
|
||||
}
|
||||
|
||||
/** One spot: its NZB read, its files listed and matched. */
|
||||
async inspect(spot: any): Promise<{ status: string; matched: number }> {
|
||||
const done = async (status: string, extra: Record<string, unknown> = {}) => {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra })
|
||||
return { status, matched: 0 }
|
||||
}
|
||||
const segments: string[] = spot.nzb_segments ?? []
|
||||
if (segments.length === 0) return done('no-nzb')
|
||||
if (segments.length > coordinatorConfig.spots.maxNzbSegments) return done('no-nzb', { error: `${segments.length} NZB articles` })
|
||||
const client = await this.connect()
|
||||
const bodies = await client.bodies(segments)
|
||||
if (bodies.some((b) => b === null)) return done('nzb-gone', { error: `${bodies.filter((b) => b === null).length} of ${segments.length} NZB articles gone` })
|
||||
let xml: string
|
||||
try {
|
||||
xml = decodeSpotBinary(bodies as Buffer[]).toString('utf8')
|
||||
} catch (error: any) {
|
||||
return done('nzb-unreadable', { error: String(error.message).slice(0, 500) })
|
||||
}
|
||||
return this.inspectNzb(spot, xml)
|
||||
}
|
||||
|
||||
/** The files of a spot's NZB listed and matched (the tests feed this directly). */
|
||||
async inspectNzb(spot: any, xml: string): Promise<{ status: string; matched: number }> {
|
||||
const nzb = parseNzb(xml)
|
||||
const finish = (status: string, extra: Record<string, unknown> = {}) =>
|
||||
db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra })
|
||||
if (!nzb || nzb.files.length === 0) {
|
||||
await finish('no-nzb')
|
||||
return { status: 'no-nzb', matched: 0 }
|
||||
}
|
||||
if (isDummyNzb(nzb, Number(spot.size ?? 0))) {
|
||||
await finish('dummy')
|
||||
return { status: 'dummy', matched: 0 }
|
||||
}
|
||||
const files: SpotFile[] = nzb.files.map((f, idx) => ({
|
||||
idx,
|
||||
name: nzbFileName(f.subject).name.slice(0, 1000),
|
||||
subject: f.subject.slice(0, 2000),
|
||||
poster: f.poster,
|
||||
groups: f.groups,
|
||||
postedAt: f.date ? new Date(f.date * 1000) : null,
|
||||
bytes: f.segments.reduce((s, x) => s + x.bytes, 0),
|
||||
segments: f.segments.length,
|
||||
}))
|
||||
// The NZB's own password over what the spot's text names.
|
||||
const password = nzb.meta.password ? { password: nzb.meta.password.slice(0, 200), source: 'nzb' } : spot.password ? { password: spot.password, source: spot.password_source } : null
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.from('spot_files').where('spot_id', spot.message_id).delete()
|
||||
for (let i = 0; i < files.length; i += 1000) {
|
||||
await trx.table('spot_files').multiInsert(
|
||||
files.slice(i, i + 1000).map((f) => ({
|
||||
spot_id: spot.message_id,
|
||||
idx: f.idx,
|
||||
name: f.name,
|
||||
subject: f.subject,
|
||||
poster: f.poster,
|
||||
groups: f.groups,
|
||||
posted_at: f.postedAt,
|
||||
bytes: f.bytes,
|
||||
segments: f.segments,
|
||||
}))
|
||||
)
|
||||
}
|
||||
await trx
|
||||
.from('spots')
|
||||
.where('message_id', spot.message_id)
|
||||
.update({
|
||||
status: 'inspected',
|
||||
inspected_at: new Date(),
|
||||
files: files.length,
|
||||
matched: 0,
|
||||
nzb: gzipSync(xml),
|
||||
...(password && password.source === 'nzb' ? { password: password.password, password_source: 'nzb' } : {}),
|
||||
})
|
||||
})
|
||||
const matched = await this.match(spot.message_id)
|
||||
return { status: 'inspected', matched }
|
||||
}
|
||||
|
||||
private async groupId(name: string): Promise<number | null> {
|
||||
if (!this.groupIds.has(name)) {
|
||||
const row = await db.from('groups').where('name', name).select('id').first()
|
||||
this.groupIds.set(name, row ? Number(row.id) : null)
|
||||
}
|
||||
return this.groupIds.get(name)!
|
||||
}
|
||||
|
||||
/**
|
||||
* Matches a spot's listed files to releases and links them; how many releases. A file's release: the same
|
||||
* identity (releaseIdentity over its subject and poster) in one of its groups, the one posted nearest the file.
|
||||
*/
|
||||
async match(spotId: string): Promise<number> {
|
||||
const spot = await db.from('spots').where('message_id', spotId).first()
|
||||
if (!spot || spot.status !== 'inspected') return 0
|
||||
const files = await db.from('spot_files').where('spot_id', spotId).orderBy('idx')
|
||||
const wanted: { idx: number; key: string; groups: number[]; at: number | null }[] = []
|
||||
for (const f of files) {
|
||||
if (!f.subject) continue
|
||||
const { name, base } = nzbFileName(f.subject)
|
||||
const identity = releaseIdentity({ poster: f.poster ?? '', subject: base, name, weak: false })
|
||||
if (!identity) continue
|
||||
const groups = (await Promise.all((f.groups ?? []).map((g: string) => this.groupId(g)))).filter((g): g is number => g !== null)
|
||||
if (groups.length > 0) wanted.push({ idx: f.idx, key: identity.key, groups, at: f.posted_at ? new Date(f.posted_at).getTime() : null })
|
||||
}
|
||||
if (wanted.length === 0) return 0
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT id, encode(identity_key, 'hex') AS identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?::bytea[]) AND group_id = ANY(?)`,
|
||||
[[...new Set(wanted.map((w) => w.key))].map((k) => identityKey(k)), [...new Set(wanted.flatMap((w) => w.groups))]]
|
||||
)
|
||||
const byKey = new Map<string, { id: number; group: number; at: number | null }[]>()
|
||||
for (const r of rows) {
|
||||
const list = byKey.get(r.identity_key) ?? []
|
||||
list.push({ id: Number(r.id), group: Number(r.group_id), at: r.posted_at ? new Date(r.posted_at).getTime() : null })
|
||||
byKey.set(r.identity_key, list)
|
||||
}
|
||||
const fileRelease = new Map<number, number>()
|
||||
for (const w of wanted) {
|
||||
const near = (byKey.get(w.key) ?? [])
|
||||
.filter((r) => w.groups.includes(r.group))
|
||||
.map((r) => ({ ...r, spread: r.at === null || w.at === null ? 0 : Math.abs(r.at - w.at) }))
|
||||
.filter((r) => r.spread <= MAX_SPREAD_MS)
|
||||
.sort((a, b) => a.spread - b.spread)
|
||||
if (near[0]) fileRelease.set(w.idx, near[0].id)
|
||||
}
|
||||
const perRelease = new Map<number, number>()
|
||||
for (const id of fileRelease.values()) perRelease.set(id, (perRelease.get(id) ?? 0) + 1)
|
||||
if (perRelease.size === 0) return 0
|
||||
await db.transaction(async (trx) => {
|
||||
// A release of the spot's own goes now that the header pass built its files' releases.
|
||||
if (spot.release_id) {
|
||||
const own = await trx.from('releases').where('id', spot.release_id).where('source', SOURCES.spot).select('id').first()
|
||||
if (own) {
|
||||
await trx.from('release_spots').where('release_id', own.id).delete()
|
||||
await trx.from('release_passwords').where('release_id', own.id).delete()
|
||||
await trx.from('releases').where('id', own.id).delete()
|
||||
}
|
||||
}
|
||||
for (const [idx, releaseId] of fileRelease) await trx.from('spot_files').where({ spot_id: spotId, idx }).update({ release_id: releaseId })
|
||||
for (const [releaseId, n] of perRelease) {
|
||||
await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: n }).onConflict(['release_id', 'spot_id']).merge({ files: n })
|
||||
}
|
||||
if (spot.password && perRelease.size > 0) {
|
||||
await trx
|
||||
.table('release_passwords')
|
||||
.multiInsert([...perRelease.keys()].map((id) => ({ release_id: id, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId })))
|
||||
.onConflict(['release_id', 'password'])
|
||||
.ignore()
|
||||
await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true })
|
||||
}
|
||||
// The spot's category for the ones whose names say nothing (migration 5200: spot_category).
|
||||
await trx.rawQuery(`UPDATE releases SET category = c FROM (SELECT spot_category(?::integer, ?::jsonb) AS c) s WHERE id = ANY(?) AND category = 8010 AND c IS NOT NULL`, [
|
||||
spot.category,
|
||||
JSON.stringify(spot.subcats ?? {}),
|
||||
[...perRelease.keys()],
|
||||
])
|
||||
const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0]
|
||||
// The NZB is only kept for a release of the spot's own: the header pass's releases have their chunks.
|
||||
await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null, nzb: null })
|
||||
})
|
||||
return perRelease.size
|
||||
}
|
||||
|
||||
/** Unmatched inspected spots matched again; the ones that waited long enough become releases of their own. */
|
||||
async rematch(): Promise<{ matched: number; standalone: number }> {
|
||||
const out = { matched: 0, standalone: 0 }
|
||||
const { rows: spots } = await db.rawQuery(
|
||||
`SELECT message_id, inspected_at, release_id FROM spots
|
||||
WHERE status = 'inspected' AND matched = 0 AND inspected_at > now() - make_interval(days => ?)
|
||||
ORDER BY rematched_at NULLS FIRST, posted_at DESC NULLS LAST LIMIT ?`,
|
||||
[REMATCH_DAYS, REMATCH_BATCH]
|
||||
)
|
||||
const cutoff = Date.now() - coordinatorConfig.spots.standaloneAfterHours * 3600 * 1000
|
||||
for (const spot of spots) {
|
||||
if (this.stopped) break
|
||||
const n = await this.match(spot.message_id)
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ rematched_at: new Date() })
|
||||
out.matched += n
|
||||
if (n === 0 && !spot.release_id && new Date(spot.inspected_at).getTime() < cutoff && (await this.standalone(spot.message_id))) out.standalone++
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/** A release of the spot's own (source `spot`), from its listed files; false when it has none. */
|
||||
async standalone(spotId: string): Promise<boolean> {
|
||||
const spot = await db.from('spots').where('message_id', spotId).first()
|
||||
const files = await db.from('spot_files').where('spot_id', spotId)
|
||||
if (!spot || files.length === 0) return false
|
||||
const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))]
|
||||
const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null
|
||||
const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex')
|
||||
// Its poster and group set by id (voyager 3, migration 6000), as the batch store stores its releases'.
|
||||
const posterId = spot.poster ? await CatalogRefs.posterId(db, spot.poster) : null
|
||||
const groupSetId = await CatalogRefs.groupSetId(db, groups)
|
||||
const [row] = await db
|
||||
.table('releases')
|
||||
.insert({
|
||||
release_key: releaseKey(key, groupId, null),
|
||||
identity_key: identityKey(key),
|
||||
group_id: groupId,
|
||||
group_set_id: groupSetId,
|
||||
poster_id: posterId,
|
||||
name: spot.title.slice(0, 1000),
|
||||
source: SOURCES.spot,
|
||||
posted_at: spot.posted_at,
|
||||
last_posted_at: spot.posted_at,
|
||||
files: files.length,
|
||||
segments: files.reduce((s: number, f: any) => s + Number(f.segments), 0),
|
||||
bytes: files.reduce((s: number, f: any) => s + Number(f.bytes), 0),
|
||||
readable: true,
|
||||
passworded: spot.password ? true : null,
|
||||
settled_at: new Date(),
|
||||
// Kept when its title says nothing (the category trigger, migration 5200).
|
||||
category: db.raw('coalesce(spot_category(?::integer, ?::jsonb), 8010)', [spot.category, JSON.stringify(spot.subcats ?? {})]),
|
||||
})
|
||||
.onConflict('release_key')
|
||||
.ignore()
|
||||
.returning('id')
|
||||
if (!row) return false
|
||||
const releaseId = Number(row.id)
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore()
|
||||
// Its title, searchable (release_texts).
|
||||
await trx.rawQuery(`INSERT INTO release_texts (release_id, kind, text) VALUES (?, 1, left(?, 255)) ON CONFLICT DO NOTHING`, [releaseId, spot.title])
|
||||
if (spot.password) {
|
||||
await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore()
|
||||
}
|
||||
await trx.from('spots').where('message_id', spotId).update({ release_id: releaseId })
|
||||
})
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -1,191 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { SOURCES } from '#lib/release/source'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import { MissingArticle, XoverClient } from '#lib/nntp/index'
|
||||
import {
|
||||
decodeSpotText,
|
||||
moderationTarget,
|
||||
parseSpotArticleHeaders,
|
||||
parseSpotHeader,
|
||||
parseSpotXml,
|
||||
passwordFromSpot,
|
||||
spotterId,
|
||||
verifySpotHeader,
|
||||
} from '#lib/spotnet/index'
|
||||
|
||||
/** Overviews asked for at a time. */
|
||||
const XOVER_STEP = 1000
|
||||
/** A personal dispose counts within this long of the spot (Spotweb). */
|
||||
const DISPOSE_WINDOW_SECONDS = 5 * 86400
|
||||
|
||||
export interface SpotterRound {
|
||||
read: number
|
||||
listed: number
|
||||
unverified: number
|
||||
moderation: number
|
||||
}
|
||||
|
||||
/**
|
||||
* The spotter (REBUILD.md, spots): Spotnet's spots, from the headers of
|
||||
* free.pt into `spots`, on one NNTP connection of its own. Every few
|
||||
* minutes it reads the overviews after the last article it saw (the first
|
||||
* time the last SPOT_BACKFILL), and for each spot the full header: its
|
||||
* signature is checked there (lib/spotnet, as Spotweb does), and the X-XML
|
||||
* gives the description, the website and the NZB's articles. A password the
|
||||
* title (`{{pw}}`) or the description names is kept with the spot; the spot
|
||||
* inspector carries it to the releases. A delete/dispose message marks its
|
||||
* target `removed` and takes back what the target gave the releases.
|
||||
*/
|
||||
export class Spotter {
|
||||
private stopped = false
|
||||
private client?: XoverClient
|
||||
|
||||
stop() {
|
||||
this.stopped = true
|
||||
this.client?.close()
|
||||
}
|
||||
|
||||
async run() {
|
||||
while (!this.stopped) {
|
||||
const started = Date.now()
|
||||
try {
|
||||
const round = await this.round()
|
||||
if (round.read > 0) {
|
||||
logger.info(`Spots: ${round.read} articles read, ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`)
|
||||
}
|
||||
} catch (error: any) {
|
||||
logger.error(`Spot round failed: ${error.message}`)
|
||||
this.client?.close()
|
||||
this.client = undefined
|
||||
}
|
||||
const wait = coordinatorConfig.spots.everySeconds * 1000
|
||||
// Quiet until the next round: closed (nntpIdleCloseMs), opened again then.
|
||||
const idleMs = coordinatorConfig.nntpIdleCloseMs
|
||||
if (idleMs > 0 && wait - (Date.now() - started) >= idleMs) {
|
||||
this.client?.close()
|
||||
this.client = undefined
|
||||
}
|
||||
for (let waited = Date.now() - started; waited < wait && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000))
|
||||
}
|
||||
}
|
||||
|
||||
private async connect(): Promise<XoverClient> {
|
||||
this.client ??= await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
return this.client
|
||||
}
|
||||
|
||||
/** The articles after the last one seen, up to the group's high. */
|
||||
async round(): Promise<SpotterRound> {
|
||||
const { group, backfill } = coordinatorConfig.spots
|
||||
const client = await this.connect()
|
||||
const { low, high } = await client.group(group)
|
||||
const state = await db.from('spot_state').where('group_name', group).first()
|
||||
let next = state ? Number(state.last_article) + 1 : Math.max(low, high - backfill + 1)
|
||||
const round: SpotterRound = { read: 0, listed: 0, unverified: 0, moderation: 0 }
|
||||
while (next <= high && !this.stopped) {
|
||||
const to = Math.min(high, next + XOVER_STEP - 1)
|
||||
const overviews = await client.xover(next, to)
|
||||
round.read += overviews.length
|
||||
for (const [number, fields] of overviews) {
|
||||
if (this.stopped) return round
|
||||
const messageId = (fields['message-id'] ?? '').replace(/^<|>$/g, '')
|
||||
if (!messageId || !parseSpotHeader(fields.from ?? '', fields.subject ?? '', messageId, fields.date)) continue
|
||||
const outcome = await this.read(client, messageId, number)
|
||||
if (outcome) round[outcome]++
|
||||
}
|
||||
await db
|
||||
.table('spot_state')
|
||||
.insert({ group_name: group, last_article: to })
|
||||
.onConflict('group_name')
|
||||
.merge({ last_article: to })
|
||||
next = to + 1
|
||||
}
|
||||
return round
|
||||
}
|
||||
|
||||
/** One spot by its full header; what it was, or null when it was there already or is gone. */
|
||||
async read(client: XoverClient, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
|
||||
if (await db.from('spots').where('message_id', messageId).first()) return null
|
||||
let raw: Buffer
|
||||
try {
|
||||
raw = await client.head(messageId)
|
||||
} catch (error) {
|
||||
if (error instanceof MissingArticle) return null
|
||||
throw error
|
||||
}
|
||||
return this.store(raw.toString('latin1'), messageId, articleNumber)
|
||||
}
|
||||
|
||||
/** Verifies and stores a spot from its raw header (the tests feed this directly). */
|
||||
async store(raw: string, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> {
|
||||
const { headers, xml: xmlText } = parseSpotArticleHeaders(raw)
|
||||
const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
|
||||
if (!header) return null
|
||||
const verified = verifySpotHeader(header)
|
||||
const xml = verified ? parseSpotXml(decodeSpotText(xmlText)) : null
|
||||
const target = verified ? moderationTarget(header) : null
|
||||
const title = xml?.title || header.title
|
||||
// Kept apart by where it came from: the title's {{pw}}, else the description's "Wachtwoord: …".
|
||||
const fromTitle = verified && !target ? passwordFromSpot(title, null) : null
|
||||
const fromText = verified && !target && !fromTitle ? passwordFromSpot('', xml?.description) : null
|
||||
const status = !verified ? 'unverified' : target ? 'moderation' : 'listed'
|
||||
await db
|
||||
.table('spots')
|
||||
.insert({
|
||||
message_id: messageId,
|
||||
article_number: articleNumber,
|
||||
title: title.slice(0, 2000),
|
||||
tag: header.tag || null,
|
||||
poster: header.poster,
|
||||
category: header.category,
|
||||
subcats: JSON.stringify(header.subcats),
|
||||
size: header.size || null,
|
||||
posted_at: header.stamp ? new Date(header.stamp * 1000) : null,
|
||||
key_id: header.keyId,
|
||||
spotter_id: header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : null,
|
||||
description: xml?.description || null,
|
||||
website: xml?.website || null,
|
||||
nzb_segments: xml?.nzbSegments.length ? xml.nzbSegments : null,
|
||||
password: fromTitle ?? fromText,
|
||||
password_source: fromTitle ? 'spot-title' : fromText ? 'spot-text' : null,
|
||||
status,
|
||||
})
|
||||
.onConflict('message_id')
|
||||
.ignore()
|
||||
if (target) await this.moderate(header.selfSignedPubKey, header.stamp, target)
|
||||
return status
|
||||
}
|
||||
|
||||
/**
|
||||
* A delete/dispose/remove: the target is `removed`, and its links and passwords go from the releases. As in
|
||||
* Spotweb, a personal dispose (the poster's own key) counts only from the same spotter within 5 days; a
|
||||
* moderator's (Spotnet's key) always.
|
||||
*/
|
||||
private async moderate(pubKey: string, stamp: number, targetId: string) {
|
||||
const target = await db.from('spots').where('message_id', targetId).first()
|
||||
if (!target) return
|
||||
const personal = pubKey.length > 50 ? spotterId(pubKey) : ''
|
||||
if (personal) {
|
||||
const age = stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
|
||||
if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
|
||||
}
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.from('spots').where('message_id', targetId).update({ status: 'removed' })
|
||||
await trx.from('release_spots').where('spot_id', targetId).delete()
|
||||
await trx.from('release_passwords').where('spot_id', targetId).delete()
|
||||
// A release the spot made of its own goes with it.
|
||||
if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', SOURCES.spot).delete()
|
||||
})
|
||||
logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
|
||||
}
|
||||
}
|
||||
@@ -1,343 +0,0 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { readdir, stat, statfs } from 'node:fs/promises'
|
||||
import { join } from 'node:path'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import GroupLeases from '#services/group_leases'
|
||||
import type { Lane } from '#lib/edge/index'
|
||||
|
||||
export interface RangeBand {
|
||||
band: string
|
||||
planned: number
|
||||
plannedArticles: number
|
||||
leased: number
|
||||
done: number
|
||||
doneArticles: number
|
||||
failed: number
|
||||
}
|
||||
|
||||
export interface HeldGroup {
|
||||
node: string
|
||||
group: string
|
||||
lane: Lane
|
||||
range: { start: number; end: number; next: number } | null
|
||||
heartbeatSeconds: number
|
||||
snapshotNext: number | null
|
||||
snapshotSeconds: number | null
|
||||
snapshotMb: number | null
|
||||
rangesDone: number
|
||||
headers: number
|
||||
}
|
||||
|
||||
export interface SnapshotFile {
|
||||
group: string
|
||||
lane: Lane
|
||||
mb: number
|
||||
hours: number
|
||||
}
|
||||
|
||||
export interface Status {
|
||||
at: string
|
||||
catalog: {
|
||||
tablespace: string
|
||||
gb: number
|
||||
capGb: number
|
||||
full: boolean
|
||||
freeGb: number
|
||||
}
|
||||
bands: RangeBand[]
|
||||
lastHour: {
|
||||
indexedArticles: number
|
||||
doneRanges: number
|
||||
storedArticles: number
|
||||
storedRanges: number
|
||||
}
|
||||
corpus: { pendingRanges: number; pendingArticles: number }
|
||||
spool: { ranges: number; chunks: number }
|
||||
tail: { groups: number; ranges: number; articles: number }
|
||||
/** The 'ancient' ranges: articles below the low watermark the groups were planned at (expired, never fetched). */
|
||||
ancient: { groups: number; articles: number }
|
||||
held: HeldGroup[]
|
||||
snapshots: { count: number; gb: number; oldestHours: number | null; largest: SnapshotFile[] }
|
||||
releases: { estimated: number; provisional: number }
|
||||
/** `ready` and `waiting` are the planner's estimates (the last analyze), the others counts. */
|
||||
probes: { ready: number; waiting: number; pendingWaiting: number; pendingNamed: number }
|
||||
judge: { dirty: number }
|
||||
nodes: { name: string; lastSeenSeconds: number | null }[]
|
||||
}
|
||||
|
||||
const seconds = (since: Date | string | null) =>
|
||||
since ? Math.round((Date.now() - new Date(since).getTime()) / 1000) : null
|
||||
|
||||
/**
|
||||
* What the indexer is doing, in one object (commands/coordinator_status.ts prints it): the catalog's room on the
|
||||
* NVMe, the ranges by band (the tail, the capture windows, the history), the last hour's rates, the held groups
|
||||
* with their snapshots, the phase-2 backlogs. Every number comes from the database, the snapshot directory or the
|
||||
* filesystem; nothing is kept between calls.
|
||||
*/
|
||||
class StatusService {
|
||||
async collect(): Promise<Status> {
|
||||
const tail = coordinatorConfig.tail.priority
|
||||
const cap = coordinatorConfig.catalogMaxNvmeGb
|
||||
// The catalog's tablespace only (CatalogSpace measures it every minute anyway): pg_database_size walks the
|
||||
// chunk partitions and the probe cache on the HDD, minutes on homelab.
|
||||
const { rows: space } = await db.rawQuery(
|
||||
`SELECT coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default') AS tablespace,
|
||||
pg_tablespace_size(coalesce(nullif(current_setting('default_tablespace'), ''), 'pg_default')) AS bytes`
|
||||
)
|
||||
const tablespaceGb = Number(space[0].bytes) / 1e9
|
||||
const fs = await statfs('/').catch(() => null)
|
||||
const freeGb = fs ? (fs.bavail * fs.bsize) / 1e9 : 0
|
||||
|
||||
// The ranges by band: the tail, the capture windows (priorities 0–2; the sampling ranges in other years are
|
||||
// capture too, at their history priorities), the history by its thousand.
|
||||
const { rows: bands } = await db.rawQuery(
|
||||
`SELECT CASE WHEN priority <= ? THEN 'tail'
|
||||
WHEN priority < 0 THEN 'raised'
|
||||
WHEN capture AND priority <= 2 THEN 'window ' || priority
|
||||
WHEN priority < 1000 THEN 'history 0-999'
|
||||
ELSE 'history ' || (priority / 1000) * 1000 || '+' END AS band,
|
||||
count(*) FILTER (WHERE status = 'planned') AS planned,
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned'), 0) AS planned_articles,
|
||||
count(*) FILTER (WHERE status = 'leased') AS leased,
|
||||
count(*) FILTER (WHERE status = 'done') AS done,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'done'), 0) AS done_articles,
|
||||
count(*) FILTER (WHERE status = 'failed') AS failed
|
||||
FROM ranges GROUP BY 1
|
||||
ORDER BY min(CASE WHEN priority <= ? THEN 0 WHEN priority < 0 THEN 1 WHEN capture AND priority <= 2 THEN 2 + priority
|
||||
WHEN priority < 1000 THEN 10 ELSE 11 + priority / 1000 END)`,
|
||||
[tail, tail]
|
||||
)
|
||||
const { rows: hour } = await db.rawQuery(
|
||||
`SELECT coalesce(sum(end_id - start_id + 1) FILTER (WHERE done_at > now() - interval '1 hour'), 0) AS indexed,
|
||||
count(*) FILTER (WHERE done_at > now() - interval '1 hour') AS done_ranges,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE stored_at > now() - interval '1 hour'), 0) AS stored,
|
||||
count(*) FILTER (WHERE stored_at > now() - interval '1 hour') AS stored_ranges,
|
||||
count(*) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done') AS corpus_pending,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE capture AND stored_at IS NULL AND status <> 'done'), 0) AS corpus_pending_articles,
|
||||
count(*) FILTER (WHERE status IN ('planned', 'leased') AND spooled_chunks > 0) AS spooled_ranges,
|
||||
coalesce(sum(spooled_chunks) FILTER (WHERE status IN ('planned', 'leased')), 0) AS spooled_chunks,
|
||||
count(DISTINCT group_id) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_groups,
|
||||
count(*) FILTER (WHERE status = 'planned' AND priority <= ?) AS tail_ranges,
|
||||
coalesce(sum(end_id - next_id + 1) FILTER (WHERE status = 'planned' AND priority <= ?), 0) AS tail_articles,
|
||||
count(DISTINCT group_id) FILTER (WHERE status = 'ancient') AS ancient_groups,
|
||||
coalesce(sum(end_id - start_id + 1) FILTER (WHERE status = 'ancient'), 0) AS ancient_articles
|
||||
FROM ranges`,
|
||||
[tail, tail, tail]
|
||||
)
|
||||
const h = hour[0]
|
||||
|
||||
const ttl = coordinatorConfig.leaseTtlSeconds
|
||||
const { rows: held } = await db.rawQuery(
|
||||
`SELECT n.name AS node, g.name AS "group", l.lane, l.heartbeat_at, l.snapshot_next, l.snapshot_at, l.ranges_done, l.headers,
|
||||
r.start_id, r.end_id, r.next_id, l.group_id
|
||||
FROM group_leases l JOIN groups g ON g.id = l.group_id JOIN nodes n ON n.id = l.node_id
|
||||
LEFT JOIN ranges r ON r.id = l.range_id
|
||||
WHERE l.node_id IS NOT NULL AND l.heartbeat_at >= now() - make_interval(secs => ?)
|
||||
ORDER BY l.lane, g.name`,
|
||||
[ttl]
|
||||
)
|
||||
const snapshotSizes = new Map<string, number>()
|
||||
const files: SnapshotFile[] = []
|
||||
const names = new Map<number, string>(
|
||||
(await db.from('groups').select('id', 'name')).map((g) => [Number(g.id), g.name as string])
|
||||
)
|
||||
for (const file of await readdir(GroupLeases.dir).catch(() => [] as string[])) {
|
||||
const m = /^(\d+)(-tail)?\.gz$/.exec(file)
|
||||
if (!m) continue
|
||||
const s = await stat(join(GroupLeases.dir, file)).catch(() => null)
|
||||
if (!s) continue
|
||||
const lane: Lane = m[2] ? 'tail' : 'history'
|
||||
snapshotSizes.set(`${m[1]}:${lane}`, s.size)
|
||||
files.push({
|
||||
group: names.get(Number(m[1])) ?? m[1],
|
||||
lane,
|
||||
mb: s.size / 1e6,
|
||||
hours: (Date.now() - s.mtimeMs) / 3600_000,
|
||||
})
|
||||
}
|
||||
files.sort((a, b) => b.mb - a.mb)
|
||||
|
||||
// The probe backlog from the planner's estimates of the partial indexes (counting 11 M rows took 10 s on
|
||||
// homelab); the rest are small partial indexes.
|
||||
const { rows: releases } = await db.rawQuery(
|
||||
`SELECT (SELECT reltuples::bigint FROM pg_class WHERE oid = 'releases'::regclass) AS estimated,
|
||||
(SELECT count(*) FROM releases WHERE settled_at IS NULL) AS provisional,
|
||||
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_ready_idx'::regclass) AS ready,
|
||||
(SELECT greatest(0, reltuples)::bigint FROM pg_class WHERE oid = 'probe_candidates_waiting_idx'::regclass) AS waiting,
|
||||
(SELECT count(*) FROM pending_releases WHERE status = 0) AS pending_waiting,
|
||||
(SELECT count(*) FROM pending_releases WHERE status = 4) AS pending_named,
|
||||
(SELECT count(*) FROM judge_dirty) AS dirty`
|
||||
)
|
||||
const r = releases[0]
|
||||
const nodes = await db.from('nodes').select('name', 'last_seen_at').orderBy('name')
|
||||
|
||||
return {
|
||||
at: new Date().toISOString(),
|
||||
catalog: {
|
||||
tablespace: space[0].tablespace,
|
||||
gb: tablespaceGb,
|
||||
capGb: cap,
|
||||
full: cap > 0 && tablespaceGb > cap,
|
||||
freeGb,
|
||||
},
|
||||
bands: bands.map((b: any) => ({
|
||||
band: b.band,
|
||||
planned: Number(b.planned),
|
||||
plannedArticles: Number(b.planned_articles),
|
||||
leased: Number(b.leased),
|
||||
done: Number(b.done),
|
||||
doneArticles: Number(b.done_articles),
|
||||
failed: Number(b.failed),
|
||||
})),
|
||||
lastHour: {
|
||||
indexedArticles: Number(h.indexed),
|
||||
doneRanges: Number(h.done_ranges),
|
||||
storedArticles: Number(h.stored),
|
||||
storedRanges: Number(h.stored_ranges),
|
||||
},
|
||||
corpus: {
|
||||
pendingRanges: Number(h.corpus_pending),
|
||||
pendingArticles: Number(h.corpus_pending_articles),
|
||||
},
|
||||
spool: { ranges: Number(h.spooled_ranges), chunks: Number(h.spooled_chunks) },
|
||||
ancient: { groups: Number(h.ancient_groups), articles: Number(h.ancient_articles) },
|
||||
tail: {
|
||||
groups: Number(h.tail_groups),
|
||||
ranges: Number(h.tail_ranges),
|
||||
articles: Number(h.tail_articles),
|
||||
},
|
||||
held: held.map((l: any) => {
|
||||
const size = snapshotSizes.get(`${l.group_id}:${l.lane}`)
|
||||
return {
|
||||
node: l.node,
|
||||
group: l.group,
|
||||
lane: l.lane,
|
||||
range:
|
||||
l.start_id !== null
|
||||
? { start: Number(l.start_id), end: Number(l.end_id), next: Number(l.next_id) }
|
||||
: null,
|
||||
heartbeatSeconds: seconds(l.heartbeat_at) ?? 0,
|
||||
snapshotNext: l.snapshot_next === null ? null : Number(l.snapshot_next),
|
||||
snapshotSeconds: seconds(l.snapshot_at),
|
||||
snapshotMb: size === undefined ? null : size / 1e6,
|
||||
rangesDone: Number(l.ranges_done),
|
||||
headers: Number(l.headers),
|
||||
}
|
||||
}),
|
||||
snapshots: {
|
||||
count: files.length,
|
||||
gb: files.reduce((sum, f) => sum + f.mb, 0) / 1000,
|
||||
oldestHours: files.length > 0 ? Math.max(...files.map((f) => f.hours)) : null,
|
||||
largest: files.slice(0, 5),
|
||||
},
|
||||
releases: { estimated: Math.max(0, Number(r.estimated)), provisional: Number(r.provisional) },
|
||||
probes: {
|
||||
ready: Number(r.ready),
|
||||
waiting: Number(r.waiting),
|
||||
pendingWaiting: Number(r.pending_waiting),
|
||||
pendingNamed: Number(r.pending_named),
|
||||
},
|
||||
judge: { dirty: Number(r.dirty) },
|
||||
nodes: nodes.map((n) => ({ name: n.name, lastSeenSeconds: seconds(n.last_seen_at) })),
|
||||
}
|
||||
}
|
||||
|
||||
/** The status as a text panel. */
|
||||
render(s: Status): string {
|
||||
const n = (v: number) => Math.round(v).toLocaleString('en-US')
|
||||
const b = (v: number) =>
|
||||
v >= 1e9
|
||||
? `${(v / 1e9).toFixed(2)} B`
|
||||
: v >= 1e6
|
||||
? `${(v / 1e6).toFixed(1)} M`
|
||||
: v >= 1e3
|
||||
? `${(v / 1e3).toFixed(0)} k`
|
||||
: String(v)
|
||||
const age = (sec: number | null) =>
|
||||
sec === null
|
||||
? '-'
|
||||
: sec < 90
|
||||
? `${sec} s`
|
||||
: sec < 5400
|
||||
? `${Math.round(sec / 60)} min`
|
||||
: `${(sec / 3600).toFixed(1)} h`
|
||||
const lines: string[] = []
|
||||
lines.push(`Status at ${s.at}`)
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Catalog: ${s.catalog.gb.toFixed(1)} GB in tablespace ${s.catalog.tablespace}${s.catalog.capGb ? ` of ${s.catalog.capGb} GB${s.catalog.full ? ' (FULL: history paused)' : ''}` : ' (no cap)'}; ${s.catalog.freeGb.toFixed(1)} GB free on /`
|
||||
)
|
||||
lines.push(
|
||||
`Last hour: ${b(s.lastHour.indexedArticles)} articles indexed in ${n(s.lastHour.doneRanges)} ranges (${n(s.lastHour.indexedArticles / 3600)}/s); corpus fill ${b(s.lastHour.storedArticles)} in ${n(s.lastHour.storedRanges)} ranges, ${n(s.corpus.pendingRanges)} capture ranges (${b(s.corpus.pendingArticles)}) still to store`
|
||||
)
|
||||
lines.push(
|
||||
`Spool: ${n(s.spool.chunks)} chunks in ${n(s.spool.ranges)} ranges. Tail: ${b(s.tail.articles)} planned in ${n(s.tail.ranges)} ranges of ${n(s.tail.groups)} groups. Ancient: ${b(s.ancient.articles)} articles below the low watermark of ${n(s.ancient.groups)} groups`
|
||||
)
|
||||
lines.push('')
|
||||
const table = (header: string[], rows: string[][]) => {
|
||||
const widths = header.map((h, i) => Math.max(h.length, ...rows.map((r) => r[i].length)))
|
||||
const line = (r: string[]) =>
|
||||
r.map((c, i) => (i === 0 ? c.padEnd(widths[i]) : c.padStart(widths[i]))).join(' ')
|
||||
lines.push(line(header))
|
||||
for (const r of rows) lines.push(line(r))
|
||||
}
|
||||
lines.push('Ranges by band:')
|
||||
table(
|
||||
['band', 'planned', 'articles', 'leased', 'done', 'articles', 'failed'],
|
||||
s.bands.map((x) => [
|
||||
x.band,
|
||||
n(x.planned),
|
||||
b(x.plannedArticles),
|
||||
n(x.leased),
|
||||
n(x.done),
|
||||
b(x.doneArticles),
|
||||
n(x.failed),
|
||||
])
|
||||
)
|
||||
lines.push('')
|
||||
lines.push(`Held groups (${s.held.length}):`)
|
||||
if (s.held.length > 0)
|
||||
table(
|
||||
[
|
||||
'group',
|
||||
'lane',
|
||||
'node',
|
||||
'range',
|
||||
'at',
|
||||
'heartbeat',
|
||||
'snapshot',
|
||||
'size',
|
||||
'done',
|
||||
'headers',
|
||||
],
|
||||
s.held.map((x) => [
|
||||
x.group,
|
||||
x.lane,
|
||||
x.node,
|
||||
x.range ? `${x.range.start}-${x.range.end}` : '-',
|
||||
x.range ? String(x.range.next) : '-',
|
||||
age(x.heartbeatSeconds),
|
||||
x.snapshotNext === null ? '-' : `${x.snapshotNext} (${age(x.snapshotSeconds)})`,
|
||||
x.snapshotMb === null ? '-' : `${x.snapshotMb.toFixed(0)} MB`,
|
||||
n(x.rangesDone),
|
||||
b(x.headers),
|
||||
])
|
||||
)
|
||||
lines.push('')
|
||||
lines.push(
|
||||
`Snapshots: ${n(s.snapshots.count)} files, ${s.snapshots.gb.toFixed(2)} GB${s.snapshots.oldestHours === null ? '' : `, oldest ${s.snapshots.oldestHours.toFixed(1)} h`}${s.snapshots.largest.length > 0 ? `; largest ${s.snapshots.largest.map((f) => `${f.group}${f.lane === 'tail' ? ' (tail)' : ''} ${f.mb.toFixed(0)} MB`).join(', ')}` : ''}`
|
||||
)
|
||||
lines.push(
|
||||
`Releases: ~${n(s.releases.estimated)}, ${n(s.releases.provisional)} provisional. Judge: ${n(s.judge.dirty)} dirty`
|
||||
)
|
||||
lines.push(
|
||||
`Probes: ~${n(s.probes.ready)} ready of ~${n(s.probes.waiting)} waiting; pending ladder ${n(s.probes.pendingWaiting)} waiting, ${n(s.probes.pendingNamed)} named`
|
||||
)
|
||||
lines.push(
|
||||
`Nodes: ${s.nodes.map((x) => `${x.name} (${age(x.lastSeenSeconds)})`).join(', ') || 'none'}`
|
||||
)
|
||||
return lines.join('\n')
|
||||
}
|
||||
}
|
||||
|
||||
export default new StatusService()
|
||||
+7
-14
@@ -11,26 +11,19 @@
|
||||
|
|
||||
*/
|
||||
|
||||
/**
|
||||
* Edge workers (edge:node) have no database or web server: what start/env
|
||||
* requires for those gets a placeholder, so a node's .env needs only the
|
||||
* coordinator's URL and its token.
|
||||
*/
|
||||
if (process.argv[2] === 'edge:node') {
|
||||
const placeholders: Record<string, string> = {
|
||||
NODE_ENV: 'production', PORT: '0', HOST: '127.0.0.1', LOG_LEVEL: 'info', APP_KEY: 'edge-node-has-no-app-key-0000000000', APP_URL: 'http://127.0.0.1',
|
||||
SESSION_DRIVER: 'memory', DB_HOST: '127.0.0.1', DB_PORT: '5432', DB_USER: 'none', DB_DATABASE: 'none',
|
||||
}
|
||||
for (const [key, value] of Object.entries(placeholders)) process.env[key] ??= value
|
||||
}
|
||||
|
||||
await import('reflect-metadata')
|
||||
const { Ignitor, prettyPrintError } = await import('@adonisjs/core/ignitor')
|
||||
const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
|
||||
|
||||
/**
|
||||
* URL to the application root. AdonisJS need it to resolve
|
||||
* paths to file and directories for scaffolding commands
|
||||
*/
|
||||
/**
|
||||
* The command being run, for start/env.ts: Ace takes the arguments out of
|
||||
* process.argv before the environment is loaded.
|
||||
*/
|
||||
process.env.ACE_COMMAND ??= process.argv[2] ?? ''
|
||||
|
||||
const APP_ROOT = new URL('../', import.meta.url)
|
||||
|
||||
/**
|
||||
|
||||
+1
-1
@@ -10,7 +10,7 @@
|
||||
*/
|
||||
|
||||
await import('reflect-metadata')
|
||||
const { Ignitor, prettyPrintError } = await import('@adonisjs/core/ignitor')
|
||||
const { Ignitor, prettyPrintError } = await import('@adonisjs/core')
|
||||
|
||||
/**
|
||||
* URL to the application root. AdonisJS need it to resolve
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@
|
||||
process.env.NODE_ENV = 'test'
|
||||
|
||||
import 'reflect-metadata'
|
||||
import { Ignitor, prettyPrintError } from '@adonisjs/core/ignitor'
|
||||
import { Ignitor, prettyPrintError } from '@adonisjs/core'
|
||||
import { configure, processCLIArgs, run } from '@japa/runner'
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NntpService from '#services/NntpService'
|
||||
import ArchiveWalkService from '#services/ArchiveWalkService'
|
||||
|
||||
const BATCH = 5000
|
||||
|
||||
/**
|
||||
* Complete archive listings for releases inspected before archive walks
|
||||
* existed (CONCEPT.md D40): queues those whose listing leaves most of the
|
||||
* release unaccounted for. worker:collection walks them, one at a time and
|
||||
* behind inspection. With release ids, walks those right here instead.
|
||||
*/
|
||||
export default class ArchiveWalk extends BaseCommand {
|
||||
public static commandName = 'content:walk-archives'
|
||||
public static description = 'Queues releases with partial archive listings for a (low priority) header walk.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@args.spread({ description: 'Release ids to walk now (instead of queueing)', required: false })
|
||||
declare releaseIds?: string[]
|
||||
|
||||
@flags.number({ description: 'Queue at most this many releases' })
|
||||
declare limit?: number
|
||||
|
||||
@flags.boolean({ description: 'Also releases that were walked before' })
|
||||
declare again: boolean
|
||||
|
||||
public async run() {
|
||||
try {
|
||||
if (this.releaseIds?.length) {
|
||||
for (const id of this.releaseIds) {
|
||||
this.logger.info(JSON.stringify(await ArchiveWalkService.walk(Number(id))))
|
||||
}
|
||||
return
|
||||
}
|
||||
let lastId = 0
|
||||
let queued = 0
|
||||
while (!this.limit || queued < this.limit) {
|
||||
const query = db
|
||||
.from('releases as r')
|
||||
.where('r.id', '>', lastId)
|
||||
.whereNotNull('r.contents')
|
||||
.orderBy('r.id')
|
||||
.limit(BATCH)
|
||||
.select('r.id', 'r.size', 'r.contents_walked_at', db.raw(`(
|
||||
SELECT count(*) FROM jsonb_array_elements(r.contents)) AS n`), db.raw(`(
|
||||
SELECT coalesce(sum((e->>'size')::bigint), 0) FROM jsonb_array_elements(r.contents) e) AS listed`))
|
||||
const rows = await query
|
||||
if (rows.length === 0) break
|
||||
lastId = Number(rows.at(-1).id)
|
||||
const jobs = rows
|
||||
.filter((r) => this.again || !r.contents_walked_at)
|
||||
.filter((r) => ArchiveWalkService.worthWalking(Number(r.n), Number(r.listed), Number(r.size)))
|
||||
.slice(0, this.limit ? this.limit - queued : undefined)
|
||||
.map((r) => ({ name: 'walk', data: { releaseId: Number(r.id) }, opts: { jobId: `walk-${r.id}-${Date.now()}` } }))
|
||||
if (jobs.length > 0) await QueueService.archiveWalkQueue.addBulk(jobs)
|
||||
queued += jobs.length
|
||||
}
|
||||
this.logger.info(`Queued ${queued} releases for an archive walk.`)
|
||||
} finally {
|
||||
await QueueService.closeAll()
|
||||
await NntpService.shutdown()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import QueueService from '#services/QueueService'
|
||||
import RedisService from '#services/RedisService'
|
||||
import DeferredService, { isDeferred } from '#services/DeferredService'
|
||||
|
||||
const BATCH = 2000
|
||||
|
||||
/** Rebuilds the wait list (KEYS[1]) without the ids in the set KEYS[2], in one atomic pass. */
|
||||
const FILTER_LIST = `
|
||||
local ids = redis.call('LRANGE', KEYS[1], 0, -1)
|
||||
local keep = {}
|
||||
for _, id in ipairs(ids) do
|
||||
if redis.call('SISMEMBER', KEYS[2], id) == 0 then keep[#keep + 1] = id end
|
||||
end
|
||||
redis.call('DEL', KEYS[1])
|
||||
for i = 1, #keep, 5000 do
|
||||
redis.call('RPUSH', KEYS[1], unpack(keep, i, math.min(i + 4999, #keep)))
|
||||
end
|
||||
return #ids - #keep`
|
||||
|
||||
/**
|
||||
* Removes waiting body jobs whose message-id is on the skip or defer list,
|
||||
* e.g. after adding a pattern (CONCEPT.md D24, D28). Deferred ones are
|
||||
* recorded in deferred_files first, so they can still be processed later.
|
||||
*/
|
||||
export default class BodyQueuePrune extends BaseCommand {
|
||||
public static commandName = 'body-queue:prune'
|
||||
public static description = 'Drops waiting body jobs that are now skipped or deferred.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only count' })
|
||||
declare dryRun: boolean
|
||||
|
||||
@flags.boolean({ description: 'Drop deferred jobs without recording them (already recorded by an earlier run)' })
|
||||
declare skipRecord: boolean
|
||||
|
||||
public async run() {
|
||||
const redis = RedisService.client
|
||||
const queue = QueueService.bodyQueue
|
||||
const waitKey = queue.toKey('wait')
|
||||
let skipped = 0
|
||||
let deferred = 0
|
||||
let scanned = 0
|
||||
const dropped: string[] = []
|
||||
try {
|
||||
const ids: string[] = await redis.lrange(waitKey, 0, -1)
|
||||
for (let i = 0; i < ids.length; i += BATCH) {
|
||||
const batch = ids.slice(i, i + BATCH)
|
||||
const data: [Error | null, string | null][] = await batch
|
||||
.reduce((pipeline, id) => pipeline.hget(queue.toKey(id), 'data'), redis.pipeline())
|
||||
.exec()
|
||||
const drop: string[] = []
|
||||
const toDefer = new Map<string, { articleNumber: number; header: Record<string, string> }[]>()
|
||||
batch.forEach((id, j) => {
|
||||
const job = data[j][1] ? JSON.parse(data[j][1]!) : null
|
||||
if (!job?.header) return
|
||||
const messageId = job.header['message-id'] ?? ''
|
||||
if (isDeferred(job.header)) {
|
||||
const list = toDefer.get(job.group) ?? []
|
||||
list.push({ articleNumber: Number(job.articleNumber), header: job.header })
|
||||
toDefer.set(job.group, list)
|
||||
drop.push(id)
|
||||
deferred++
|
||||
} else if (indexerConfig.skipBodyMessageIds.some((re) => re.test(messageId))) {
|
||||
drop.push(id)
|
||||
skipped++
|
||||
}
|
||||
})
|
||||
scanned += batch.length
|
||||
if (this.dryRun || drop.length === 0) continue
|
||||
if (!this.skipRecord) for (const [group, articles] of toDefer) await DeferredService.record(group, articles)
|
||||
dropped.push(...drop)
|
||||
}
|
||||
if (dropped.length > 0) {
|
||||
const dropKey = queue.toKey('prune-ids')
|
||||
await redis.del(dropKey)
|
||||
for (let i = 0; i < dropped.length; i += BATCH) await redis.sadd(dropKey, ...dropped.slice(i, i + BATCH))
|
||||
await redis.eval(FILTER_LIST, 2, waitKey, dropKey)
|
||||
await redis.del(dropKey)
|
||||
for (let i = 0; i < dropped.length; i += BATCH) {
|
||||
await redis.unlink(...dropped.slice(i, i + BATCH).map((id) => queue.toKey(id)))
|
||||
}
|
||||
}
|
||||
this.logger.info(
|
||||
`Scanned ${scanned} waiting body jobs: ${this.dryRun ? 'would drop' : 'dropped'} ${deferred} deferred and ${skipped} skipped.`
|
||||
)
|
||||
} finally {
|
||||
await QueueService.closeAll()
|
||||
await RedisService.close()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import QueueService from '#services/QueueService'
|
||||
import ContentService from '#services/ContentService'
|
||||
import { isArchiveStart } from '#lib/content/index'
|
||||
import InspectionQueue, { INSPECTION_PRIORITY } from '#services/InspectionQueue'
|
||||
|
||||
const BATCH = 1000
|
||||
|
||||
/**
|
||||
* Queues completed files for content inspection (names and groupings from
|
||||
* par2 and RAR headers, CONCEPT.md D21) that were completed before it
|
||||
* existed. Costs one article download per file, done by worker:collection.
|
||||
*/
|
||||
export default class ContentBackfill extends BaseCommand {
|
||||
public static commandName = 'content:backfill'
|
||||
public static description = 'Queues existing files with random names for content inspection.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.number({ description: 'Queue at most this many files' })
|
||||
declare limit?: number
|
||||
|
||||
@flags.boolean({ description: 'Also files that were inspected before (after release:rebuild --reset-content)' })
|
||||
declare all: boolean
|
||||
|
||||
@flags.boolean({ description: 'Only archives whose listing contains NZBs, to import those (CONCEPT.md D28)' })
|
||||
declare archiveNzbs: boolean
|
||||
|
||||
@flags.boolean({ description: 'Only unlisted 7z archives (first volumes: .7z, .7z.001)' })
|
||||
declare sevenZip: boolean
|
||||
|
||||
@flags.boolean({ description: 'Archive starts never inspected (RAR first volumes, 7z, .bin), also files from NZBs/spots' })
|
||||
declare archives: boolean
|
||||
|
||||
@flags.boolean({ description: 'SFV files not read yet (their listings group the files they name, D34)' })
|
||||
declare sfv: boolean
|
||||
|
||||
@flags.boolean({ description: 'niconico captures not read yet (their releases get the video title, D34)' })
|
||||
declare nicovideo: boolean
|
||||
|
||||
public async run() {
|
||||
let lastId = 0
|
||||
let queued = 0
|
||||
try {
|
||||
while (!this.limit || queued < this.limit) {
|
||||
// Files never inspected: no fingerprint on their binary and no archive listing.
|
||||
const query = db.from('files as f').leftJoin('file_details as d', 'd.file_id', 'f.id').where('f.id', '>', lastId)
|
||||
// Files from NZBs (spots) have no binary.
|
||||
if (this.sevenZip || this.archives || this.sfv || this.nicovideo) query.leftJoin('binaries as b', 'b.file_id', 'f.id')
|
||||
else query.join('binaries as b', 'b.file_id', 'f.id')
|
||||
if (this.nicovideo) {
|
||||
query.whereNull('d.content_type').whereRaw(`f.filename ~* '^\\d{9,11}_nicovideo_jp_watch_sm\\d+\\.tar\\.zst$'`)
|
||||
} else if (this.sfv) {
|
||||
query.whereNull('d.content_type').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.sfv$'`)
|
||||
} else if (this.archives) {
|
||||
query.whereNull('d.content_type').whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.(rar|7z|7z\\.0*1|bin)$'`)
|
||||
} else if (this.sevenZip) {
|
||||
query.whereNull('d.archive').whereRaw(`coalesce(f.real_name, f.filename) ~* '\\.7z(\\.0*1){0,1}$'`)
|
||||
} else if (this.archiveNzbs) query.whereRaw(`d.archive::text ~* '\\.nzb"'`)
|
||||
else if (!this.all) query.whereNull('b.md5_16k').whereNull('d.archive')
|
||||
const files = await query
|
||||
.orderBy('f.id')
|
||||
.limit(BATCH)
|
||||
.select('f.id', 'f.filename')
|
||||
if (files.length === 0) break
|
||||
lastId = Number(files.at(-1).id)
|
||||
|
||||
const jobs = files
|
||||
.filter((f) => (this.archives ? isArchiveStart(f.filename) : this.sfv || this.nicovideo || ContentService.isCandidate(f.filename)))
|
||||
.slice(0, this.limit ? this.limit - queued : undefined)
|
||||
.map((f) => Number(f.id))
|
||||
// Into the inspection backlog (InspectionQueue): targeted backfills go ahead of what's waiting;
|
||||
// niconico captures are many (~190k) and only cosmetic, so behind everything else.
|
||||
const priority = this.nicovideo
|
||||
? INSPECTION_PRIORITY.low
|
||||
: this.archiveNzbs || this.sevenZip || this.archives || this.sfv
|
||||
? INSPECTION_PRIORITY.targeted
|
||||
: INSPECTION_PRIORITY.normal
|
||||
if (jobs.length > 0) await InspectionQueue.add(jobs, priority)
|
||||
queued += jobs.length
|
||||
}
|
||||
this.logger.info(`Queued ${queued} files for content inspection.`)
|
||||
} finally {
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import IndexMaintenance from '#services/IndexMaintenance'
|
||||
|
||||
/** Rebuilds bloated indexes online (gc:scheduler also does this weekly). */
|
||||
export default class DbReindex extends BaseCommand {
|
||||
public static commandName = 'db:reindex'
|
||||
public static description = 'Rebuilds B-tree indexes that are mostly empty space (online, REINDEX CONCURRENTLY).'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only show which indexes would be rebuilt' })
|
||||
declare dryRun: boolean
|
||||
|
||||
public async run() {
|
||||
const rebuilt = await IndexMaintenance.rebuildBloated({ dryRun: this.dryRun })
|
||||
const saved = rebuilt.reduce((sum, r) => sum + r.before - r.after, 0)
|
||||
this.logger.info(`${rebuilt.length} indexes ${this.dryRun ? 'would be rebuilt' : `rebuilt, ${Math.round(saved / 1024 / 1024)} MB freed`}.`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import DeferredService from '#services/DeferredService'
|
||||
|
||||
/**
|
||||
* Sends recorded deferred ranges (fully obfuscated ngPost articles, CONCEPT.md
|
||||
* D28) to the body queue, oldest first. Every article in them is downloaded:
|
||||
* their headers can't be grouped into files.
|
||||
*/
|
||||
export default class DeferredExpandRanges extends BaseCommand {
|
||||
public static commandName = 'deferred:expand-ranges'
|
||||
public static description = 'Queues the bodies of deferred article ranges (costly: every article is downloaded).'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.number({ description: 'Ranges to expand', default: 10 })
|
||||
declare limit: number
|
||||
|
||||
public async run() {
|
||||
try {
|
||||
const { ranges, articles } = await DeferredService.expandRanges(this.limit)
|
||||
this.logger.info(`Expanded ${ranges} ranges: ${articles} articles sent to the body queue.`)
|
||||
} finally {
|
||||
await NntpService.shutdown()
|
||||
await QueueService.closeAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { unquotedFileName } from '#services/NzbImportService'
|
||||
|
||||
/**
|
||||
* Files named after a whole NZB subject without quotes (spot NZBs), e.g.
|
||||
* `Adventure.part0055.rar - [056/175] (1/1)`: renamed to the filename in it.
|
||||
*/
|
||||
export default class FilesCleanNames extends BaseCommand {
|
||||
public static commandName = 'files:clean-names'
|
||||
public static description = 'Strips subject leftovers (counters, "yEnc") from file names that came from NZBs.'
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
@flags.boolean({ description: 'Only show what would change' })
|
||||
declare dryRun: boolean
|
||||
|
||||
public async run() {
|
||||
const files = await db
|
||||
.from('files')
|
||||
.whereRaw(`filename ~* '(\\(\\d+/\\d+\\)|yEnc|\\[\\d+/\\d+\\])\\s*$'`)
|
||||
.select('id', 'filename')
|
||||
const changes = files
|
||||
.map((f) => ({ id: Number(f.id), from: f.filename as string, to: unquotedFileName(f.filename) }))
|
||||
.filter((c) => c.to !== c.from)
|
||||
for (const c of changes.slice(0, 5)) this.logger.info(`"${c.from}" → "${c.to}"`)
|
||||
if (!this.dryRun) {
|
||||
for (const c of changes) await db.from('files').where('id', c.id).update({ filename: c.to })
|
||||
}
|
||||
this.logger.info(`${changes.length} of ${files.length} file names ${this.dryRun ? 'would change' : 'cleaned'}.`)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
import { BaseCommand } from '@adonisjs/core/ace'
|
||||
import { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
|
||||
const BATCH = 50_000
|
||||
|
||||
/**
|
||||
* One-off (LOG.md 2026-10-08 (14)): copies the content types and archive
|
||||
* listings still on `files` into file_details, a partition at a time.
|
||||
* Values written since the deploy (by the new code, into file_details) win:
|
||||
* a file that has details already is skipped.
|
||||
*/
|
||||
export default class FilesMoveDetails extends BaseCommand {
|
||||
public static commandName = 'files:move-details'
|
||||
public static description = "Copies files' content types and archive listings into file_details."
|
||||
|
||||
public static options: CommandOptions = {
|
||||
startApp: true,
|
||||
}
|
||||
|
||||
public async run() {
|
||||
const { rows: partitions } = await db.rawQuery(
|
||||
`SELECT c.relname FROM pg_inherits i JOIN pg_class c ON c.oid = i.inhrelid WHERE i.inhparent = 'files'::regclass ORDER BY c.relname`
|
||||
)
|
||||
let total = 0
|
||||
for (const { relname } of partitions) {
|
||||
const started = Date.now()
|
||||
const { rows } = await db.rawQuery(`SELECT min(id) AS lo, max(id) AS hi FROM ${relname}`)
|
||||
if (rows[0].lo === null) continue
|
||||
let copied = 0
|
||||
// Short statements in id order: one statement over a whole partition took 30 minutes, and the
|
||||
// garbage filter deleted files meanwhile, which failed the foreign key (LOG.md 2026-10-08 (15)).
|
||||
for (let lo = Number(rows[0].lo); lo <= Number(rows[0].hi); lo += BATCH) {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
const result = await db.rawQuery(
|
||||
`INSERT INTO file_details (file_id, content_type, archive)
|
||||
SELECT id, content_type, archive FROM ${relname}
|
||||
WHERE id >= ? AND id < ? AND (content_type IS NOT NULL OR archive IS NOT NULL)
|
||||
ON CONFLICT (file_id) DO NOTHING`,
|
||||
[lo, lo + BATCH]
|
||||
)
|
||||
copied += result.rowCount ?? 0
|
||||
break
|
||||
} catch (error: any) {
|
||||
// A file deleted while this batch ran: the batch again, without it.
|
||||
if (error.code !== '23503' || attempt >= 5) throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
total += copied
|
||||
this.logger.info(`${relname}: ${copied} copied in ${Math.round((Date.now() - started) / 1000)} s.`)
|
||||
}
|
||||
this.logger.info(`${total} files' details copied.`)
|
||||
}
|
||||
}
|
||||
Loaded 100 of 532 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user