Compare commits
19
Commits
No files matched your search
@@ -19,6 +19,11 @@ DB_USER=usenet_indexer
|
|||||||
DB_PASSWORD=
|
DB_PASSWORD=
|
||||||
DB_DATABASE=usenet_rebuild
|
DB_DATABASE=usenet_rebuild
|
||||||
|
|
||||||
|
# Access (config/access.ts): open (NEWZNAB_API_KEY or none, no limits) or accounts
|
||||||
|
# (a Newznab key per user, tiers with daily limits; ace access:invite, access:tier, access:suspend)
|
||||||
|
# ACCESS=open
|
||||||
|
# SIGNUP=open|invite|closed (with accounts: invite unless set)
|
||||||
|
|
||||||
#--------------------------------------------------------------------
|
#--------------------------------------------------------------------
|
||||||
# CORS (configure allowed origins for API access)
|
# CORS (configure allowed origins for API access)
|
||||||
#--------------------------------------------------------------------
|
#--------------------------------------------------------------------
|
||||||
|
|||||||
@@ -272,6 +272,15 @@ spot, a day after settling). Verdicts (`release_verdicts`, with the rule
|
|||||||
set's version) hide releases from search (`judged` shows them); a verdict
|
set's version) hide releases from search (`judged` shows them); a verdict
|
||||||
whose rule no longer matches is taken back. Nothing is dropped yet.
|
whose rule no longer matches is taken back. Nothing is dropped yet.
|
||||||
|
|
||||||
|
**Sidecars** (the merger's third pass). An upload's .nfo, .sfv or .nzb
|
||||||
|
that came apart from it (another name, another counter: "[00/80] … .nzb"
|
||||||
|
beside "[01/94] … .part01.rar") was a release of its own. The settler
|
||||||
|
queues small settled releases (≤ 3 files, < 1 MB) in `fold_candidates`;
|
||||||
|
one whose files are all sidecars joins the poster's bigger release in the
|
||||||
|
same group posted within 10 minutes: the one sharing a word of 4+ letters
|
||||||
|
with its subject, else the one under the same counter total, else none
|
||||||
|
(`release_merges.via` 'sidecar').
|
||||||
|
|
||||||
**Spots** (after voyager 1's D29; `lib/spotnet` ported as is). Two
|
**Spots** (after voyager 1's D29; `lib/spotnet` ported as is). Two
|
||||||
processes, one NNTP connection each:
|
processes, one NNTP connection each:
|
||||||
|
|
||||||
|
|||||||
@@ -1,13 +1,36 @@
|
|||||||
import User from '#models/user'
|
import User from '#models/user'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
import { signupValidator } from '#validators/user'
|
import { signupValidator } from '#validators/user'
|
||||||
import type { HttpContext } from '@adonisjs/core/http'
|
import type { HttpContext } from '@adonisjs/core/http'
|
||||||
import UserTransformer from '#transformers/user_transformer'
|
import UserTransformer from '#transformers/user_transformer'
|
||||||
|
import AccessService from '#services/access_service'
|
||||||
|
|
||||||
|
class InviteGone extends Error {}
|
||||||
|
|
||||||
export default class NewAccountController {
|
export default class NewAccountController {
|
||||||
async store({ request, serialize }: HttpContext) {
|
async store({ request, response, serialize }: HttpContext) {
|
||||||
const { fullName, email, password } = await request.validateUsing(signupValidator)
|
const { fullName, email, password, invite } = await request.validateUsing(signupValidator)
|
||||||
|
if (AccessService.signup === 'closed') return response.forbidden({ error: 'Signup is closed' })
|
||||||
|
if (AccessService.signup === 'invite' && !invite) return response.forbidden({ error: 'An invite code is needed to sign up' })
|
||||||
|
|
||||||
const user = await User.create({ fullName, email, password })
|
// The user and their invite together: an invite that's gone rolls the signup back.
|
||||||
|
const user = await db
|
||||||
|
.transaction(async (trx) => {
|
||||||
|
const created = await User.create({ fullName, email, password }, { client: trx })
|
||||||
|
if (AccessService.signup !== 'invite') return created
|
||||||
|
const gives = await AccessService.redeem(invite!, created.id, trx)
|
||||||
|
if (!gives) throw new InviteGone()
|
||||||
|
if (gives.tier) {
|
||||||
|
AccessService.grantTier(created, gives.tier, gives.tier_days)
|
||||||
|
await created.useTransaction(trx).save()
|
||||||
|
}
|
||||||
|
return created
|
||||||
|
})
|
||||||
|
.catch((error) => {
|
||||||
|
if (error instanceof InviteGone) return null
|
||||||
|
throw error
|
||||||
|
})
|
||||||
|
if (!user) return response.forbidden({ error: 'Unknown or used invite code' })
|
||||||
const token = await User.accessTokens.create(user)
|
const token = await User.accessTokens.create(user)
|
||||||
|
|
||||||
return serialize({
|
return serialize({
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ import type { HttpContext } from '@adonisjs/core/http'
|
|||||||
import env from '#start/env'
|
import env from '#start/env'
|
||||||
import NzbService from '#services/nzb_service'
|
import NzbService from '#services/nzb_service'
|
||||||
import SearchService from '#services/search_service'
|
import SearchService from '#services/search_service'
|
||||||
|
import AccessService, { AccessDenied } from '#services/access_service'
|
||||||
import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
|
import { capsXml, errorXml, expandCategories, resultsXml, type NewznabServer } from '#lib/newznab/index'
|
||||||
|
|
||||||
const LIMITS = { max: 100, default: 50 }
|
const LIMITS = { max: 100, default: 50 }
|
||||||
@@ -35,22 +36,37 @@ function int(value: unknown): number | undefined {
|
|||||||
* Releases with readable names (or an inner name from the prober, which then is the title) that aren't known to miss files.
|
* Releases with readable names (or an inner name from the prober, which then is the title) that aren't known to miss files.
|
||||||
*/
|
*/
|
||||||
export default class NewznabController {
|
export default class NewznabController {
|
||||||
async handle({ request, response }: HttpContext) {
|
async handle(ctx: HttpContext) {
|
||||||
|
const xml = (body: string, type = 'application/xml') => {
|
||||||
|
ctx.response.header('Content-Type', `${type}; charset=utf-8`)
|
||||||
|
return ctx.response.send(body)
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return await this.answer(ctx, xml)
|
||||||
|
} catch (error) {
|
||||||
|
if (error instanceof AccessDenied) return xml(errorXml(error.newznabCode, error.message))
|
||||||
|
throw error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async answer(ctx: HttpContext, xml: (body: string, type?: string) => void) {
|
||||||
|
const { request, response } = ctx
|
||||||
const qs = request.qs()
|
const qs = request.qs()
|
||||||
const t = String(qs.t ?? '')
|
const t = String(qs.t ?? '')
|
||||||
const server = this.server(request)
|
|
||||||
const xml = (body: string, type = 'application/xml') => {
|
|
||||||
response.header('Content-Type', `${type}; charset=utf-8`)
|
|
||||||
return response.send(body)
|
|
||||||
}
|
|
||||||
|
|
||||||
const apiKey = configuredApiKey()
|
// With accounts the key is a user's (AccessService); else the one configured, if any.
|
||||||
if (apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
|
const grant = await AccessService.authorize(ctx, qs.apikey)
|
||||||
if (t === 'caps') return xml(capsXml(server, LIMITS))
|
const apiKey = grant ? grant.user.apiKey : configuredApiKey()
|
||||||
|
if (!grant && apiKey && qs.apikey !== apiKey) return xml(errorXml(100, 'Incorrect user credentials'))
|
||||||
|
const server = this.server(request, apiKey)
|
||||||
|
if (t === 'caps') return xml(capsXml(server, LIMITS, { available: AccessService.accounts && AccessService.signup !== 'closed', open: AccessService.accounts && AccessService.signup === 'open' }))
|
||||||
|
|
||||||
if (t === 'get') {
|
if (t === 'get') {
|
||||||
const id = int(qs.id)
|
const id = int(qs.id)
|
||||||
if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
|
if (id === undefined) return xml(errorXml(200, 'Missing parameter (id)'))
|
||||||
|
// Unknown ids and releases past the tier's retention aren't counted.
|
||||||
|
if (!(await AccessService.checkRelease(grant, id))) return xml(errorXml(300, 'No such item'))
|
||||||
|
await AccessService.charge(grant, 'grab')
|
||||||
const nzb = await NzbService.build(id)
|
const nzb = await NzbService.build(id)
|
||||||
if (!nzb) return xml(errorXml(300, 'No such item'))
|
if (!nzb) return xml(errorXml(300, 'No such item'))
|
||||||
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
|
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
|
||||||
@@ -59,6 +75,7 @@ export default class NewznabController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
|
if (!(t in SEARCH_FUNCTIONS)) return xml(errorXml(202, `No such function (${t || 'none'})`))
|
||||||
|
await AccessService.charge(grant, 'api')
|
||||||
|
|
||||||
const requested = String(qs.cat ?? '')
|
const requested = String(qs.cat ?? '')
|
||||||
.split(',')
|
.split(',')
|
||||||
@@ -79,7 +96,7 @@ export default class NewznabController {
|
|||||||
categories,
|
categories,
|
||||||
readable: true,
|
readable: true,
|
||||||
notIncomplete: true,
|
notIncomplete: true,
|
||||||
maxAgeDays: int(qs.maxage),
|
maxAgeDays: AccessService.maxAgeDays(grant, int(qs.maxage)),
|
||||||
season: int(qs.season),
|
season: int(qs.season),
|
||||||
episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
|
episode: int(qs.season) !== undefined ? int(qs.ep) : undefined,
|
||||||
sort: 'newest',
|
sort: 'newest',
|
||||||
@@ -105,12 +122,20 @@ export default class NewznabController {
|
|||||||
episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
|
episode: tv ? `E${tv[2].padStart(2, '0')}` : undefined,
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
return xml(resultsXml(server, items, offset, result?.total ?? 0), 'application/rss+xml')
|
// What's left of the day, for clients that show it (newznab:apilimits).
|
||||||
|
const limits = grant && {
|
||||||
|
apiCurrent: grant.usage.apiHits,
|
||||||
|
apiMax: grant.limits.apiHits,
|
||||||
|
grabCurrent: grant.usage.grabs,
|
||||||
|
grabMax: grant.limits.grabs,
|
||||||
|
}
|
||||||
|
return xml(resultsXml(server, items, offset, result?.total ?? 0, limits || undefined), 'application/rss+xml')
|
||||||
}
|
}
|
||||||
|
|
||||||
private server(request: HttpContext['request']): NewznabServer {
|
/** Download links carry the caller's key. */
|
||||||
|
private server(request: HttpContext['request'], apiKey: string | undefined): NewznabServer {
|
||||||
const proto = request.header('x-forwarded-proto') ?? 'http'
|
const proto = request.header('x-forwarded-proto') ?? 'http'
|
||||||
const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
|
const host = request.header('x-forwarded-host') ?? request.header('host') ?? 'localhost'
|
||||||
return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey: configuredApiKey() }
|
return { title: 'Voyager 2', baseUrl: `${proto}://${host}`, apiKey }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,8 +1,19 @@
|
|||||||
import UserTransformer from '#transformers/user_transformer'
|
import UserTransformer from '#transformers/user_transformer'
|
||||||
import type { HttpContext } from '@adonisjs/core/http'
|
import type { HttpContext } from '@adonisjs/core/http'
|
||||||
|
import AccessService from '#services/access_service'
|
||||||
|
|
||||||
export default class ProfileController {
|
export default class ProfileController {
|
||||||
|
/** The user, their tier, Newznab key and what they used today. */
|
||||||
async show({ auth, serialize }: HttpContext) {
|
async show({ auth, serialize }: HttpContext) {
|
||||||
return serialize(UserTransformer.transform(auth.use('api').getUserOrFail()))
|
const user = auth.use('api').getUserOrFail()
|
||||||
|
return serialize({ user: UserTransformer.transform(user), access: await AccessService.summary(user) })
|
||||||
|
}
|
||||||
|
|
||||||
|
/** POST /api/v1/account/api-key: a new Newznab key (the old one stops working). */
|
||||||
|
async resetApiKey({ auth }: HttpContext) {
|
||||||
|
const user = auth.use('api').getUserOrFail()
|
||||||
|
user.apiKey = AccessService.newKey()
|
||||||
|
await user.save()
|
||||||
|
return { apiKey: user.apiKey }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
import type { HttpContext } from '@adonisjs/core/http'
|
import type { HttpContext } from '@adonisjs/core/http'
|
||||||
import NzbService from '#services/nzb_service'
|
import NzbService from '#services/nzb_service'
|
||||||
|
import AccessService from '#services/access_service'
|
||||||
import SearchService, { type SortOrder } from '#services/search_service'
|
import SearchService, { type SortOrder } from '#services/search_service'
|
||||||
import { expandCategories } from '#lib/newznab/index'
|
import { expandCategories } from '#lib/newznab/index'
|
||||||
|
|
||||||
@@ -14,13 +15,16 @@ function megabytes(value: unknown): number | undefined {
|
|||||||
/** The catalog for the web interface (REBUILD.md, part 9). */
|
/** The catalog for the web interface (REBUILD.md, part 9). */
|
||||||
export default class ReleasesController {
|
export default class ReleasesController {
|
||||||
/** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&unlocked=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
|
/** GET /api/v1/releases?q=&cat=5000,2040&readable=1&complete=1&unlocked=1&min_size=&max_size=(MB)&source=&poster=&sort=&page=&per_page= */
|
||||||
async index({ request }: HttpContext) {
|
async index(ctx: HttpContext) {
|
||||||
const qs = request.qs()
|
const qs = ctx.request.qs()
|
||||||
|
const grant = await AccessService.authorize(ctx, qs.apikey)
|
||||||
|
await AccessService.charge(grant, 'api')
|
||||||
const categories = String(qs.cat ?? '')
|
const categories = String(qs.cat ?? '')
|
||||||
.split(',')
|
.split(',')
|
||||||
.map((c) => Number(c.trim()))
|
.map((c) => Number(c.trim()))
|
||||||
.filter((c) => Number.isInteger(c) && c > 0)
|
.filter((c) => Number.isInteger(c) && c > 0)
|
||||||
return SearchService.search({
|
return SearchService.search({
|
||||||
|
maxAgeDays: AccessService.maxAgeDays(grant),
|
||||||
q: typeof qs.q === 'string' ? qs.q : undefined,
|
q: typeof qs.q === 'string' ? qs.q : undefined,
|
||||||
categories: categories.length ? expandCategories(categories) : undefined,
|
categories: categories.length ? expandCategories(categories) : undefined,
|
||||||
readable: flag(qs.readable),
|
readable: flag(qs.readable),
|
||||||
@@ -39,13 +43,20 @@ export default class ReleasesController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** GET /api/v1/releases/:id, with its files */
|
/** GET /api/v1/releases/:id, with its files */
|
||||||
async show({ params, response }: HttpContext) {
|
async show(ctx: HttpContext) {
|
||||||
const release = await SearchService.release(Number(params.id))
|
const grant = await AccessService.authorize(ctx, ctx.request.qs().apikey)
|
||||||
return release ?? response.notFound({ error: 'Release not found' })
|
if (!(await AccessService.checkRelease(grant, Number(ctx.params.id)))) return ctx.response.notFound({ error: 'Release not found' })
|
||||||
|
await AccessService.charge(grant, 'api')
|
||||||
|
const release = await SearchService.release(Number(ctx.params.id))
|
||||||
|
return release ?? ctx.response.notFound({ error: 'Release not found' })
|
||||||
}
|
}
|
||||||
|
|
||||||
/** GET /api/v1/releases/:id/nzb */
|
/** GET /api/v1/releases/:id/nzb (?apikey= for a plain link, as the web interface's) */
|
||||||
async nzb({ params, response }: HttpContext) {
|
async nzb(ctx: HttpContext) {
|
||||||
|
const { params, response } = ctx
|
||||||
|
const grant = await AccessService.authorize(ctx, ctx.request.qs().apikey)
|
||||||
|
if (!(await AccessService.checkRelease(grant, Number(params.id)))) return response.notFound({ error: 'Release not found' })
|
||||||
|
await AccessService.charge(grant, 'grab')
|
||||||
const nzb = await NzbService.build(Number(params.id))
|
const nzb = await NzbService.build(Number(params.id))
|
||||||
if (!nzb) return response.notFound({ error: 'Release not found' })
|
if (!nzb) return response.notFound({ error: 'Release not found' })
|
||||||
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
|
const filename = `${nzb.name.replace(/[^\w.\-()[\] ]+/g, '_').slice(0, 180) || 'release'}.nzb`
|
||||||
@@ -54,7 +65,12 @@ export default class ReleasesController {
|
|||||||
return response.send(nzb.xml)
|
return response.send(nzb.xml)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** GET /api/v1/stats */
|
/** GET /api/v1/access: whether the web interface needs an account, and how one is made. */
|
||||||
|
async access() {
|
||||||
|
return { mode: AccessService.mode, signup: AccessService.signup }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** GET /api/v1/stats (open: what the index holds, not what's in it) */
|
||||||
async stats() {
|
async stats() {
|
||||||
return SearchService.stats()
|
return SearchService.stats()
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,6 +8,9 @@ export default class HttpExceptionHandler extends ExceptionHandler {
|
|||||||
*/
|
*/
|
||||||
protected debug = !app.inProduction
|
protected debug = !app.inProduction
|
||||||
|
|
||||||
|
/** A refused request (AccessService: no key, a limit reached) is an answer, not a fault. */
|
||||||
|
protected ignoreCodes = ['E_ACCESS_DENIED']
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The method is used for handling errors and returning
|
* The method is used for handling errors and returning
|
||||||
* response to the client
|
* response to the client
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
|
import { randomBytes } from 'node:crypto'
|
||||||
import { UserSchema } from '#database/schema'
|
import { UserSchema } from '#database/schema'
|
||||||
|
import { beforeCreate } from '@adonisjs/lucid/orm'
|
||||||
import hash from '@adonisjs/core/services/hash'
|
import hash from '@adonisjs/core/services/hash'
|
||||||
import { compose } from '@adonisjs/core/helpers'
|
import { compose } from '@adonisjs/core/helpers'
|
||||||
import { withAuthFinder } from '@adonisjs/auth/mixins/lucid'
|
import { withAuthFinder } from '@adonisjs/auth/mixins/lucid'
|
||||||
@@ -8,6 +10,12 @@ export default class User extends compose(UserSchema, withAuthFinder(hash)) {
|
|||||||
static accessTokens = DbAccessTokensProvider.forModel(User)
|
static accessTokens = DbAccessTokensProvider.forModel(User)
|
||||||
declare currentAccessToken?: AccessToken
|
declare currentAccessToken?: AccessToken
|
||||||
|
|
||||||
|
/** The Newznab key (AccessService), known on the model right away rather than from the column's default. */
|
||||||
|
@beforeCreate()
|
||||||
|
static assignApiKey(user: User) {
|
||||||
|
user.apiKey ??= randomBytes(16).toString('hex')
|
||||||
|
}
|
||||||
|
|
||||||
get initials() {
|
get initials() {
|
||||||
const [first, last] = this.fullName ? this.fullName.split(' ') : this.email.split('@')
|
const [first, last] = this.fullName ? this.fullName.split(' ') : this.email.split('@')
|
||||||
if (first && last) {
|
if (first && last) {
|
||||||
|
|||||||
@@ -0,0 +1,171 @@
|
|||||||
|
import { randomBytes } from 'node:crypto'
|
||||||
|
import { DateTime } from 'luxon'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import { Exception } from '@adonisjs/core/exceptions'
|
||||||
|
import type { HttpContext } from '@adonisjs/core/http'
|
||||||
|
import type { TransactionClientContract } from '@adonisjs/lucid/types/database'
|
||||||
|
import User from '#models/user'
|
||||||
|
import { ACCESS, SIGNUP, TIERS, type TierLimits } from '#config/access'
|
||||||
|
|
||||||
|
export type Use = 'api' | 'grab'
|
||||||
|
|
||||||
|
/** Newznab's error codes for access (the catalog answers with the HTTP status). */
|
||||||
|
const STATUS = { 100: 401, 101: 403, 102: 403, 500: 429, 501: 429 } as const
|
||||||
|
|
||||||
|
/** Why a request is refused; the catalog's answer (Newznab catches it for its own). */
|
||||||
|
export class AccessDenied extends Exception {
|
||||||
|
constructor(
|
||||||
|
readonly newznabCode: keyof typeof STATUS,
|
||||||
|
message: string
|
||||||
|
) {
|
||||||
|
super(message, { status: STATUS[newznabCode], code: 'E_ACCESS_DENIED' })
|
||||||
|
}
|
||||||
|
|
||||||
|
async handle(error: this, ctx: HttpContext) {
|
||||||
|
ctx.response.status(error.status).send({ error: error.message, code: error.newznabCode })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Usage {
|
||||||
|
apiHits: number
|
||||||
|
grabs: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A caller let in: who, their tier as it is now, and what they used today. */
|
||||||
|
export interface Grant {
|
||||||
|
user: User
|
||||||
|
tier: string
|
||||||
|
limits: TierLimits
|
||||||
|
usage: Usage
|
||||||
|
}
|
||||||
|
|
||||||
|
const COLUMNS: Record<Use, string> = { api: 'api_hits', grab: 'grabs' }
|
||||||
|
const TODAY = `(now() AT TIME ZONE 'UTC')::date`
|
||||||
|
const DAY_MS = 86_400_000
|
||||||
|
|
||||||
|
/** Accounts, tiers and their daily limits (config/access.ts). */
|
||||||
|
export default class AccessService {
|
||||||
|
/** From config/access.ts; tests switch it. */
|
||||||
|
static mode: 'open' | 'accounts' = ACCESS
|
||||||
|
static signup: 'open' | 'invite' | 'closed' = SIGNUP
|
||||||
|
|
||||||
|
static get accounts() {
|
||||||
|
return this.mode === 'accounts'
|
||||||
|
}
|
||||||
|
|
||||||
|
static newKey() {
|
||||||
|
return randomBytes(16).toString('hex')
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The user's tier as it is now: one that ran out (or isn't configured) is free. */
|
||||||
|
static tierOf(user: User): { tier: string; limits: TierLimits } {
|
||||||
|
const expired = user.tierUntil !== null && user.tierUntil.toMillis() <= Date.now()
|
||||||
|
const tier = !expired && TIERS[user.tier] ? user.tier : 'free'
|
||||||
|
return { tier, limits: TIERS[tier] }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The user a request names: the web interface's token, else an apikey (Newznab, the catalog's NZB links). */
|
||||||
|
static async caller(ctx: HttpContext, apikey?: unknown): Promise<User | null> {
|
||||||
|
const guard = ctx.auth.use('api')
|
||||||
|
if (guard.isAuthenticated || (await guard.check())) return guard.user!
|
||||||
|
if (typeof apikey === 'string' && /^[0-9a-f]{32}$/.test(apikey)) return User.findBy('apiKey', apikey)
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Lets a request in (counting nothing yet); null when access is open. */
|
||||||
|
static async authorize(ctx: HttpContext, apikey?: unknown): Promise<Grant | null> {
|
||||||
|
if (!this.accounts) return null
|
||||||
|
const user = await this.caller(ctx, apikey)
|
||||||
|
if (!user) throw new AccessDenied(100, 'Incorrect user credentials')
|
||||||
|
if (user.suspended) throw new AccessDenied(101, 'Account suspended')
|
||||||
|
return { user, ...this.tierOf(user), usage: await this.usage(user.id) }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Counts one use against the grant's tier, or refuses it once the day's limit is reached. */
|
||||||
|
static async charge(grant: Grant | null, use: Use) {
|
||||||
|
if (!grant) return
|
||||||
|
const usage = await this.count(grant.user.id, use, use === 'api' ? grant.limits.apiHits : grant.limits.grabs)
|
||||||
|
if (!usage) throw use === 'api' ? new AccessDenied(500, 'Request limit reached') : new AccessDenied(501, 'Download limit reached')
|
||||||
|
grant.usage = usage
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Adds one use to today's row unless the limit is reached (null then): one
|
||||||
|
* statement, so concurrent requests can't both take the last one.
|
||||||
|
*/
|
||||||
|
static async count(userId: number, use: Use, limit: number | null): Promise<Usage | null> {
|
||||||
|
if (limit !== null && limit <= 0) return null
|
||||||
|
const column = COLUMNS[use]
|
||||||
|
const { rows } = await db.rawQuery(
|
||||||
|
`INSERT INTO usage (user_id, day, ${column}) VALUES (?, ${TODAY}, 1)
|
||||||
|
ON CONFLICT (user_id, day) DO UPDATE SET ${column} = usage.${column} + 1
|
||||||
|
${limit === null ? '' : `WHERE usage.${column} < ?`}
|
||||||
|
RETURNING api_hits, grabs`,
|
||||||
|
limit === null ? [userId] : [userId, limit]
|
||||||
|
)
|
||||||
|
return rows[0] ? { apiHits: rows[0].api_hits, grabs: rows[0].grabs } : null
|
||||||
|
}
|
||||||
|
|
||||||
|
static async usage(userId: number): Promise<Usage> {
|
||||||
|
const { rows } = await db.rawQuery(`SELECT api_hits, grabs FROM usage WHERE user_id = ? AND day = ${TODAY}`, [userId])
|
||||||
|
return { apiHits: rows[0]?.api_hits ?? 0, grabs: rows[0]?.grabs ?? 0 }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A search's age bound: the asked one within the tier's retention. */
|
||||||
|
static maxAgeDays(grant: Grant | null, requested?: number): number | undefined {
|
||||||
|
const retention = grant?.limits.retentionDays
|
||||||
|
if (!retention) return requested
|
||||||
|
return requested ? Math.min(requested, retention) : retention
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Refuses a release posted before the tier's retention (one without a date is let through). */
|
||||||
|
static checkPosted(grant: Grant | null, postedAt: Date | string | null) {
|
||||||
|
const retention = grant?.limits.retentionDays
|
||||||
|
if (!retention || !postedAt) return
|
||||||
|
if (new Date(postedAt).getTime() < Date.now() - retention * DAY_MS) {
|
||||||
|
throw new AccessDenied(102, `Older than your tier's retention (${retention} days)`)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether a release exists, refusing it when it's older than the tier sees; asked before an NZB is counted. */
|
||||||
|
static async checkRelease(grant: Grant | null, id: number): Promise<boolean> {
|
||||||
|
const release = await db.from('releases').where('id', id).select('posted_at').first()
|
||||||
|
if (!release) return false
|
||||||
|
this.checkPosted(grant, release.posted_at)
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Takes an unused invite for a new user: what it gives, or null when it's unknown or used. */
|
||||||
|
static async redeem(code: string, userId: number, trx: TransactionClientContract) {
|
||||||
|
const { rows } = await trx.rawQuery(
|
||||||
|
`UPDATE invites SET used_by = ?, used_at = now() WHERE code = ? AND used_by IS NULL AND used_at IS NULL RETURNING tier, tier_days`,
|
||||||
|
[userId, code]
|
||||||
|
)
|
||||||
|
return (rows[0] as { tier: string | null; tier_days: number | null } | undefined) ?? null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Gives a user a tier for `days` (null: without end). The same tier still
|
||||||
|
* running is extended from where it ends, so a renewal paid early loses nothing.
|
||||||
|
*/
|
||||||
|
static grantTier(user: User, tier: string, days: number | null) {
|
||||||
|
if (!TIERS[tier]) throw new Error(`No such tier: ${tier} (config/access.ts has ${Object.keys(TIERS).join(', ')})`)
|
||||||
|
const now = DateTime.utc()
|
||||||
|
const running = user.tier === tier && user.tierUntil && user.tierUntil > now ? user.tierUntil : now
|
||||||
|
user.tier = tier
|
||||||
|
user.tierUntil = days === null ? null : running.plus({ days })
|
||||||
|
}
|
||||||
|
|
||||||
|
/** What a profile shows of the account's access. */
|
||||||
|
static async summary(user: User) {
|
||||||
|
const { tier, limits } = this.tierOf(user)
|
||||||
|
return {
|
||||||
|
mode: this.mode,
|
||||||
|
tier,
|
||||||
|
tierUntil: tier === user.tier ? user.tierUntil : null,
|
||||||
|
apiKey: user.apiKey,
|
||||||
|
limits,
|
||||||
|
usage: await this.usage(user.id),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -95,6 +95,9 @@ class BatchStore {
|
|||||||
.update({
|
.update({
|
||||||
next_id: batch.next,
|
next_id: batch.next,
|
||||||
headers: db.raw('headers + ?', [batch.headers]),
|
headers: db.raw('headers + ?', [batch.headers]),
|
||||||
|
// The range's posting time span (release_final): the batches' trimmed spans, min and max (least and greatest skip nulls).
|
||||||
|
posted_low: db.raw('least(posted_low, to_timestamp(?::double precision / 1000))', [batch.postedLow ?? null]),
|
||||||
|
posted_high: db.raw('greatest(posted_high, to_timestamp(?::double precision / 1000))', [batch.postedHigh ?? null]),
|
||||||
heartbeat_at: new Date(),
|
heartbeat_at: new Date(),
|
||||||
})
|
})
|
||||||
if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1)) throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
|
if (Number(moved) !== 1 && !(Array.isArray(moved) && moved.length === 1)) throw new StaleBatch(`Range ${rangeId} moved on meanwhile`)
|
||||||
@@ -240,6 +243,9 @@ class BatchStore {
|
|||||||
taken.set(r.key, n)
|
taken.set(r.key, n)
|
||||||
return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
|
return { release_key: n === 1 ? r.key : `${r.key}:${n}`, key: r.key, upload: r.upload, ...row(r) }
|
||||||
}
|
}
|
||||||
|
// Their posters, for poster search (posters: one row per poster, searched by trigram; migration 4000).
|
||||||
|
const posters = [...new Set(fresh.map((r) => r.poster).filter((p): p is string => !!p))]
|
||||||
|
if (posters.length > 0) await trx.rawQuery(`INSERT INTO posters (poster) SELECT DISTINCT left(p, 200) FROM unnest(?::text[]) AS p ON CONFLICT DO NOTHING`, [posters])
|
||||||
let inserts = fresh.map(numbered)
|
let inserts = fresh.map(numbered)
|
||||||
// Writers of other groups number the same key at the same time (a spam post crossposted to 67
|
// Writers of other groups number the same key at the same time (a spam post crossposted to 67
|
||||||
// groups; only the group is locked): a number taken meanwhile is skipped, and the next one tried.
|
// groups; only the group is locked): a number taken meanwhile is skipped, and the next one tried.
|
||||||
|
|||||||
@@ -83,7 +83,7 @@ class FragmentMerger {
|
|||||||
}
|
}
|
||||||
const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
|
const releases: { rangeId: number; chunk: ReleaseChunk; candidates: ProbeCandidate[] }[] = []
|
||||||
for (const [key, { files, rangeId }] of byRelease) {
|
for (const [key, { files, rangeId }] of byRelease) {
|
||||||
const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId })))
|
const verdict = rules.judge(key, files.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster })))
|
||||||
if (verdict.rule !== null) continue
|
if (verdict.rule !== null) continue
|
||||||
uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
|
uploadsOf(files).forEach((list, upload) => releases.push({ rangeId, chunk: chunkOf(key, list, upload), candidates: candidatesOf(key, list, upload) }))
|
||||||
}
|
}
|
||||||
|
|||||||
+11
-1
@@ -17,6 +17,13 @@ export class LeaseLost extends Error {}
|
|||||||
* prefetched (LOG.md 2026-10-09). A lease whose worker went quiet for
|
* prefetched (LOG.md 2026-10-09). A lease whose worker went quiet for
|
||||||
* LEASE_TTL_SECONDS is taken over and resumes at its resume point.
|
* LEASE_TTL_SECONDS is taken over and resumes at its resume point.
|
||||||
*/
|
*/
|
||||||
|
/**
|
||||||
|
* A group's batches wait for each other (the batch store locks the group): more workers than this on one group only
|
||||||
|
* queue. Eight on one picture group's gap ranges (priority -1, in id order) held the node API's batches for minutes
|
||||||
|
* (LOG.md 2026-10-16 (68)).
|
||||||
|
*/
|
||||||
|
const MAX_LEASES_PER_GROUP = 2
|
||||||
|
|
||||||
class Leases {
|
class Leases {
|
||||||
async lease(nodeId: number): Promise<Lease | null> {
|
async lease(nodeId: number): Promise<Lease | null> {
|
||||||
const rules = await RuleSets.current()
|
const rules = await RuleSets.current()
|
||||||
@@ -29,9 +36,12 @@ class Leases {
|
|||||||
SELECT id FROM ranges
|
SELECT id FROM ranges
|
||||||
WHERE ${spooled ? 'spooled_chunks > 0' : 'true'} AND next_id <= end_id ${history}
|
WHERE ${spooled ? 'spooled_chunks > 0' : 'true'} AND next_id <= end_id ${history}
|
||||||
AND (status = 'planned' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))
|
AND (status = 'planned' OR (status = 'leased' AND heartbeat_at < now() - make_interval(secs => ?)))
|
||||||
|
AND (SELECT count(*) FROM ranges l
|
||||||
|
WHERE l.status = 'leased' AND l.group_id = ranges.group_id AND l.id <> ranges.id
|
||||||
|
AND l.heartbeat_at >= now() - make_interval(secs => ?)) < ${MAX_LEASES_PER_GROUP}
|
||||||
ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
ORDER BY priority, id LIMIT 1 FOR UPDATE SKIP LOCKED)
|
||||||
RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
|
RETURNING r.id, r.group_id, r.start_id, r.end_id, r.next_id`,
|
||||||
[nodeId, rules.version, coordinatorConfig.leaseTtlSeconds]
|
[nodeId, rules.version, coordinatorConfig.leaseTtlSeconds, coordinatorConfig.leaseTtlSeconds]
|
||||||
)
|
)
|
||||||
let { rows } = await take(true)
|
let { rows } = await take(true)
|
||||||
if (rows.length === 0) ({ rows } = await take(false))
|
if (rows.length === 0) ({ rows } = await take(false))
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ import db from '@adonisjs/lucid/services/db'
|
|||||||
import logger from '@adonisjs/core/services/logger'
|
import logger from '@adonisjs/core/services/logger'
|
||||||
import { par2ReleaseName } from '#lib/release/index'
|
import { par2ReleaseName } from '#lib/release/index'
|
||||||
import { looksRandom } from '#lib/release/random'
|
import { looksRandom } from '#lib/release/random'
|
||||||
|
import { SidecarFolder } from '#services/sidecar_folder'
|
||||||
|
import { Par2Pruner } from '#services/par2_pruner'
|
||||||
|
|
||||||
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
|
/** Releases of a set further apart than this (posted) aren't one upload: a repost is merged separately (decision d). */
|
||||||
const MAX_SPREAD_MS = 48 * 3600 * 1000
|
const MAX_SPREAD_MS = 48 * 3600 * 1000
|
||||||
@@ -17,6 +19,12 @@ const SLACK_FILES = 10
|
|||||||
const ROUND_MS = 60_000
|
const ROUND_MS = 60_000
|
||||||
/** A round looks back this far past the previous one's start (rows committed while it ran). */
|
/** A round looks back this far past the previous one's start (rows committed while it ran). */
|
||||||
const OVERLAP_MS = 60_000
|
const OVERLAP_MS = 60_000
|
||||||
|
/** The sidecar fold also runs this often inside a long par2 pass: its queue grows by the settler's pace. */
|
||||||
|
const FOLD_EVERY_MS = 5 * 60_000
|
||||||
|
/** Between par2_files prunes (complete and judged sets' file lists). */
|
||||||
|
const PRUNE_EVERY_MS = 60 * 60_000
|
||||||
|
/** The merger's row in worker_cursors: the start of its last whole round, kept over a restart. */
|
||||||
|
const CURSOR = 'merger'
|
||||||
|
|
||||||
export interface MergeRound {
|
export interface MergeRound {
|
||||||
sets: number
|
sets: number
|
||||||
@@ -26,6 +34,8 @@ export interface MergeRound {
|
|||||||
stems: number
|
stems: number
|
||||||
stemMerges: number
|
stemMerges: number
|
||||||
stemReleases: number
|
stemReleases: number
|
||||||
|
/** The sidecar fold: candidates looked at, sidecar-only ones among them, folded into their upload. */
|
||||||
|
folded?: { looked: number; sidecars: number; folded: number }
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -69,19 +79,29 @@ interface Member {
|
|||||||
*/
|
*/
|
||||||
export class Par2Merger {
|
export class Par2Merger {
|
||||||
private stopped = false
|
private stopped = false
|
||||||
private since = new Date(0)
|
private folder = new SidecarFolder((trx, found, key, via) => this.merge(trx, found, key, via))
|
||||||
|
/** Null until read from worker_cursors (1970 the first time: everything). */
|
||||||
|
private since: Date | null = null
|
||||||
|
|
||||||
stop() {
|
stop() {
|
||||||
this.stopped = true
|
this.stopped = true
|
||||||
}
|
}
|
||||||
|
|
||||||
async run() {
|
async run() {
|
||||||
|
let prunedAt = 0
|
||||||
while (!this.stopped) {
|
while (!this.stopped) {
|
||||||
const started = Date.now()
|
const started = Date.now()
|
||||||
try {
|
try {
|
||||||
|
// First, at start too: a prune needn't wait for a long round.
|
||||||
|
if (!this.stopped && Date.now() - prunedAt >= PRUNE_EVERY_MS) {
|
||||||
|
prunedAt = Date.now()
|
||||||
|
const pruned = await new Par2Pruner().round()
|
||||||
|
if (pruned.rows) logger.info(`Pruned ${pruned.rows} par2 file rows: ${pruned.complete} complete sets, ${pruned.judged} judged.`)
|
||||||
|
}
|
||||||
const round = await this.round()
|
const round = await this.round()
|
||||||
if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
|
if (round.merges > 0) logger.info(`Merged ${round.releases} releases into ${round.merges} by par2 set (${round.sets} sets checked).`)
|
||||||
if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
|
if (round.stemMerges > 0) logger.info(`Merged ${round.stemReleases} releases into ${round.stemMerges} by inner name (${round.stems} stems checked).`)
|
||||||
|
if (round.folded?.folded) logger.info(`Folded ${round.folded.folded} sidecars into their uploads (${round.folded.sidecars} sidecar-only of ${round.folded.looked} small releases).`)
|
||||||
} catch (error: any) {
|
} catch (error: any) {
|
||||||
logger.error(`Par2 merge round failed: ${error.message}`)
|
logger.error(`Par2 merge round failed: ${error.message}`)
|
||||||
}
|
}
|
||||||
@@ -92,16 +112,34 @@ export class Par2Merger {
|
|||||||
/** The sets with something new since the last round: a par2 found, or a hash of one of their files. */
|
/** The sets with something new since the last round: a par2 found, or a hash of one of their files. */
|
||||||
async round(): Promise<MergeRound> {
|
async round(): Promise<MergeRound> {
|
||||||
const { rows: now } = await db.rawQuery(`SELECT now() AS at`)
|
const { rows: now } = await db.rawQuery(`SELECT now() AS at`)
|
||||||
|
if (!this.since) {
|
||||||
|
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
|
||||||
|
this.since = saved ? new Date(saved.at) : new Date(0)
|
||||||
|
}
|
||||||
const since = new Date(this.since.getTime() - OVERLAP_MS)
|
const since = new Date(this.since.getTime() - OVERLAP_MS)
|
||||||
|
// First: sidecars (.nfo, .sfv, .nzb) apart from their upload join it. A queue, not since: it needn't wait for the
|
||||||
|
// par2 pass (hours when that starts over).
|
||||||
|
const folded = await this.folder.round()
|
||||||
|
let foldedAt = Date.now()
|
||||||
|
const foldAgain = async () => {
|
||||||
|
if (Date.now() - foldedAt < FOLD_EVERY_MS) return
|
||||||
|
const more = await this.folder.round()
|
||||||
|
folded.looked += more.looked
|
||||||
|
folded.sidecars += more.sidecars
|
||||||
|
folded.folded += more.folded
|
||||||
|
foldedAt = Date.now()
|
||||||
|
if (more.folded) logger.info(`Folded ${more.folded} sidecars into their uploads (${more.sidecars} sidecar-only of ${more.looked} small releases).`)
|
||||||
|
}
|
||||||
const { rows } = await db.rawQuery(
|
const { rows } = await db.rawQuery(
|
||||||
`SELECT set_id FROM par2_sets WHERE created_at >= ?
|
`SELECT set_id FROM par2_sets WHERE created_at >= ?
|
||||||
UNION
|
UNION
|
||||||
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size WHERE h.created_at >= ?`,
|
SELECT f.set_id FROM file_hashes h JOIN par2_files f ON f.hash16k = h.hash16k AND f.length = h.size WHERE h.created_at >= ?`,
|
||||||
[since, since]
|
[since, since]
|
||||||
)
|
)
|
||||||
const round: MergeRound = { sets: rows.length, merges: 0, releases: 0, stems: 0, stemMerges: 0, stemReleases: 0 }
|
const round: MergeRound = { sets: rows.length, merges: 0, releases: 0, stems: 0, stemMerges: 0, stemReleases: 0, folded }
|
||||||
for (const { set_id: setId } of rows) {
|
for (const { set_id: setId } of rows) {
|
||||||
if (this.stopped) return round
|
if (this.stopped) return round
|
||||||
|
await foldAgain()
|
||||||
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
|
// One set that fails doesn't hold up the others (nor the next round, which would start from the same point).
|
||||||
const merged = await this.mergeSet(setId.trim()).catch((error) => {
|
const merged = await this.mergeSet(setId.trim()).catch((error) => {
|
||||||
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
|
logger.error(`Par2 merge of set ${setId.trim()} failed: ${error.message}`)
|
||||||
@@ -129,7 +167,12 @@ export class Par2Merger {
|
|||||||
round.stemReleases += merged + 1
|
round.stemReleases += merged + 1
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Only after a whole round: a restart halfway goes on from the round before.
|
||||||
this.since = now[0].at
|
this.since = now[0].at
|
||||||
|
await db.rawQuery(
|
||||||
|
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||||
|
[CURSOR, this.since]
|
||||||
|
)
|
||||||
return round
|
return round
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -207,7 +250,7 @@ export class Par2Merger {
|
|||||||
* Merges the found releases into the one with the most files (named by the par2 set's files when none
|
* Merges the found releases into the one with the most files (named by the par2 set's files when none
|
||||||
* is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
|
* is readable); how many went into it. Each merge is kept in release_merges, with its set or stem.
|
||||||
*/
|
*/
|
||||||
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner'): Promise<number> {
|
private async merge(trx: any, found: Map<string, Member>, key: string, via: 'par2' | 'inner' | 'sidecar'): Promise<number> {
|
||||||
{
|
{
|
||||||
// Locked in id order (the batch store and other rounds take the same rows).
|
// Locked in id order (the batch store and other rounds take the same rows).
|
||||||
const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
|
const { rows: locked } = await trx.rawQuery(`SELECT id FROM releases WHERE id = ANY(?) ORDER BY id FOR UPDATE`, [[...found.keys()]])
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
|
||||||
|
/** A set this long past its last holder's par2 has had its merges: one posted later is a repost (decision d). */
|
||||||
|
const SETTLED_MS = 2 * 86400_000
|
||||||
|
/** Sets per statement. */
|
||||||
|
const BATCH = 20_000
|
||||||
|
/** The complete pass's row in worker_cursors: the newest holder's created_at of the last set it looked at. */
|
||||||
|
const CURSOR = 'par2-prune'
|
||||||
|
/** Verdicts whose releases needn't merge any more; locked isn't one (a password found takes it back). */
|
||||||
|
const JUNK = ['dead', 'fake', 'repeated']
|
||||||
|
|
||||||
|
export interface PruneRound {
|
||||||
|
complete: number
|
||||||
|
judged: number
|
||||||
|
rows: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drops par2_files rows the merge can't use any more (LOG.md 2026-10-16 (60)): the table only grew, 35M rows and
|
||||||
|
* 13.6 GB at 1.8M sets. A set's file list matches file hashes to releases and names an unreadable upload once; it's
|
||||||
|
* done when
|
||||||
|
* - complete: SETTLED_MS after its last holder's par2, every file it describes has its hash in a release holding the
|
||||||
|
* set (all of it merged into one);
|
||||||
|
* - judged: every release holding it has a junk verdict (dead, fake, repeated).
|
||||||
|
* par2_sets stays (the judge's unusable rule, the merge's holders). A set comes up again only with a new holder: its
|
||||||
|
* par2 reposted, whose probe writes the rows again; without rows its merge finds no files and the count check holds.
|
||||||
|
*/
|
||||||
|
export class Par2Pruner {
|
||||||
|
async round(): Promise<PruneRound> {
|
||||||
|
const round: PruneRound = { complete: 0, judged: 0, rows: 0 }
|
||||||
|
const saved = await db.from('worker_cursors').where('name', CURSOR).first()
|
||||||
|
let since: Date = saved ? new Date(saved.at) : new Date(0)
|
||||||
|
const until = new Date(Date.now() - SETTLED_MS)
|
||||||
|
for (;;) {
|
||||||
|
const { rows: sets } = await db.rawQuery(
|
||||||
|
`SELECT set_id, max(created_at) AS at FROM par2_sets GROUP BY set_id
|
||||||
|
HAVING max(created_at) > ? AND max(created_at) <= ? ORDER BY 2 LIMIT ?`,
|
||||||
|
[since, until, BATCH]
|
||||||
|
)
|
||||||
|
if (sets.length === 0) break
|
||||||
|
const ids = sets.map((s: { set_id: string }) => s.set_id)
|
||||||
|
const { rows: done } = await db.rawQuery(
|
||||||
|
`SELECT c.set_id FROM unnest(?::char(32)[]) AS c(set_id)
|
||||||
|
WHERE EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM par2_files f WHERE f.set_id = c.set_id AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM file_hashes h JOIN par2_sets s ON s.release_id = h.release_id AND s.set_id = f.set_id
|
||||||
|
WHERE h.hash16k = f.hash16k AND h.size = f.length))`,
|
||||||
|
[ids]
|
||||||
|
)
|
||||||
|
round.complete += done.length
|
||||||
|
round.rows += await this.drop(done.map((s: { set_id: string }) => s.set_id))
|
||||||
|
since = sets.at(-1).at
|
||||||
|
await db.rawQuery(
|
||||||
|
`INSERT INTO worker_cursors (name, at) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET at = excluded.at`,
|
||||||
|
[CURSOR, since]
|
||||||
|
)
|
||||||
|
if (sets.length < BATCH) break
|
||||||
|
}
|
||||||
|
// Judged: no cursor (verdicts come and go); only sets with rows left.
|
||||||
|
const { rows: judged } = await db.rawQuery(
|
||||||
|
`SELECT DISTINCT s.set_id FROM release_verdicts v JOIN par2_sets s ON s.release_id = v.release_id
|
||||||
|
WHERE v.rule = ANY(?)
|
||||||
|
AND EXISTS (SELECT 1 FROM par2_files f WHERE f.set_id = s.set_id)
|
||||||
|
AND NOT EXISTS (
|
||||||
|
SELECT 1 FROM par2_sets o LEFT JOIN release_verdicts ov ON ov.release_id = o.release_id AND ov.rule = ANY(?)
|
||||||
|
WHERE o.set_id = s.set_id AND ov.release_id IS NULL)`,
|
||||||
|
[JUNK, JUNK]
|
||||||
|
)
|
||||||
|
round.judged = judged.length
|
||||||
|
const judgedIds = judged.map((s: { set_id: string }) => s.set_id)
|
||||||
|
for (let i = 0; i < judgedIds.length; i += BATCH) round.rows += await this.drop(judgedIds.slice(i, i + BATCH))
|
||||||
|
return round
|
||||||
|
}
|
||||||
|
|
||||||
|
private async drop(setIds: string[]): Promise<number> {
|
||||||
|
if (setIds.length === 0) return 0
|
||||||
|
const result = await db.rawQuery(`DELETE FROM par2_files WHERE set_id = ANY(?::char(32)[])`, [setIds])
|
||||||
|
return result.rowCount ?? 0
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* range_blocks (migration 3900): per group, each run of consecutive done ranges with its posting time span, read by
|
||||||
|
* release_final(). A few thousand rows from ~650k ranges, rebuilt whole in one statement each time; a run breaks at a
|
||||||
|
* range that isn't done or doesn't start where the one before ended.
|
||||||
|
*/
|
||||||
|
class RangeBlocks {
|
||||||
|
async rebuild(): Promise<number> {
|
||||||
|
return db.transaction(async (trx) => {
|
||||||
|
await trx.rawQuery(`DELETE FROM range_blocks`)
|
||||||
|
const result = await trx.rawQuery(
|
||||||
|
`WITH r AS (
|
||||||
|
SELECT group_id, start_id, end_id, status = 'done' AS done, posted_low, posted_high,
|
||||||
|
min(start_id) OVER (PARTITION BY group_id) AS group_first,
|
||||||
|
lag(status = 'done') OVER w AS done_before, lag(end_id) OVER w AS end_before
|
||||||
|
FROM ranges WINDOW w AS (PARTITION BY group_id ORDER BY start_id)),
|
||||||
|
runs AS (
|
||||||
|
SELECT *, sum(CASE WHEN done IS DISTINCT FROM done_before OR end_before IS DISTINCT FROM start_id - 1 THEN 1 ELSE 0 END)
|
||||||
|
OVER (PARTITION BY group_id ORDER BY start_id) AS run
|
||||||
|
FROM r)
|
||||||
|
INSERT INTO range_blocks (group_id, start_id, end_id, posted_low, posted_high, at_group_start)
|
||||||
|
SELECT group_id, min(start_id), max(end_id), min(posted_low), max(posted_high), min(start_id) = min(group_first)
|
||||||
|
FROM runs WHERE done GROUP BY group_id, run`
|
||||||
|
)
|
||||||
|
return result.rowCount ?? 0
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export default new RangeBlocks()
|
||||||
@@ -8,9 +8,15 @@ const ROUND_MS = 60 * 60 * 1000
|
|||||||
/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
|
/** A file's first 16 KiB and size in this many releases is one file posted again and again (test files, spam). */
|
||||||
const REPEATED_RELEASES = 20
|
const REPEATED_RELEASES = 20
|
||||||
|
|
||||||
/** A name that says it's a film, an episode or music: a resolution, a codec, a rip source, a media extension. */
|
/**
|
||||||
|
* A name that says it's a film, an episode or music: a resolution, a codec, a rip source, a media extension. No "?" in
|
||||||
|
* these patterns: a raw query takes it for a binding ("h[ .]?26[45]" and "web[ .-]?dl" never matched; LOG.md
|
||||||
|
* 2026-10-16 (64)), so {0,1}.
|
||||||
|
*/
|
||||||
const MEDIA =
|
const MEDIA =
|
||||||
`(^|[^a-z0-9])(s[0-9]{1,2}e[0-9]{1,3}|[0-9]{3,4}p|x26[45]|h[ .]?26[45]|hevc|xvid|divx|bluray|bdrip|brrip|web[ .-]?dl|webrip|hdtv|dvdrip|dvdr|remux|mkv|mp4|avi|wmv|m4v|flac|mp3)([^a-z0-9]|$)`
|
`(^|[^a-z0-9])(s[0-9]{1,2}e[0-9]{1,3}|[0-9]{3,4}p|x26[45]|h[ .]{0,1}26[45]|hevc|xvid|divx|bluray|bdrip|brrip|web[ .-]{0,1}dl|webrip|hdtv|dvdrip|dvdr|remux|mkv|mp4|avi|wmv|m4v|flac|mp3)([^a-z0-9]|$)`
|
||||||
|
/** The posters of scraped celebrity pictures. */
|
||||||
|
const SCRAPED_POSTERS = `celeb[-_ ]{0,1}movie|celebwow|celebfx|iluvcelebs|nude[-_ ]{0,1}celebs|celeb-sex-tapes`
|
||||||
/** An executable's name. */
|
/** An executable's name. */
|
||||||
const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
|
const EXECUTABLE = `\\.(exe|scr|com|bat|cmd|msi|lnk|vbs|js|jar|pif|cpl|hta|ps1|dll)$`
|
||||||
|
|
||||||
@@ -52,6 +58,15 @@ export const RULES: Rule[] = [
|
|||||||
sql: `SELECT DISTINCT h.release_id FROM file_hashes h
|
sql: `SELECT DISTINCT h.release_id FROM file_hashes h
|
||||||
JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
|
JOIN (SELECT hash16k, size FROM file_hashes GROUP BY 1, 2 HAVING count(DISTINCT release_id) >= ${REPEATED_RELEASES}) x USING (hash16k, size)`,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: 'scraped',
|
||||||
|
says: 'celebrity pictures scraped from the web, reposted by one spam family',
|
||||||
|
// "@_CelebMovie_@", "NudeCelebs@CelebMovie.com", "@_CelebWow_@", "@_CelebFX_@", "@_iLuvCelebs_@", "@nude-celebs", …:
|
||||||
|
// a celebrity's scraped pictures as a folder under a typed subject, the same files again and again under other
|
||||||
|
// titles (LOG.md 2026-10-16 (64)). Not Celeborn, CelebTA or other posters with "celeb" in their name.
|
||||||
|
sql: `SELECT r.id FROM releases r
|
||||||
|
WHERE r.poster ~* '${SCRAPED_POSTERS}' AND r.source = 'subject' AND r.file_total IS NULL`,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
name: 'locked',
|
name: 'locked',
|
||||||
says: 'passworded, and no spot, NZB or title gives the password',
|
says: 'passworded, and no spot, NZB or title gives the password',
|
||||||
@@ -59,9 +74,11 @@ export const RULES: Rule[] = [
|
|||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: 'unusable',
|
name: 'unusable',
|
||||||
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling',
|
says: 'random names, no name inside, no par2 or inner-name merge, no spot, a day after settling, and final',
|
||||||
|
// Final (release_final): next to an unread range its par2 or the rest of its files may still come (LOG.md 2026-10-16 (67)).
|
||||||
sql: `SELECT r.id FROM releases r
|
sql: `SELECT r.id FROM releases r
|
||||||
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
|
WHERE r.settled_at < now() - interval '1 day' AND NOT r.readable AND r.inner_name IS NULL
|
||||||
|
AND release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete)
|
||||||
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
|
AND NOT EXISTS (SELECT 1 FROM release_merges m WHERE m.into_id = r.id)
|
||||||
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
|
AND NOT EXISTS (SELECT 1 FROM release_spots s WHERE s.release_id = r.id)
|
||||||
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
|
AND NOT EXISTS (SELECT 1 FROM par2_sets p WHERE p.release_id = r.id)`,
|
||||||
|
|||||||
@@ -51,6 +51,8 @@ export interface ReleaseSummary {
|
|||||||
size: number
|
size: number
|
||||||
readable: boolean
|
readable: boolean
|
||||||
complete: boolean | null
|
complete: boolean | null
|
||||||
|
/** The complete verdict is definite: done ranges cover the release's posting times and 6 hours either side (release_final). */
|
||||||
|
final: boolean
|
||||||
category: number
|
category: number
|
||||||
settled: boolean
|
settled: boolean
|
||||||
/** The prober found encrypted archive entries or headers; null: not probed (yet). */
|
/** The prober found encrypted archive entries or headers; null: not probed (yet). */
|
||||||
@@ -59,6 +61,8 @@ export interface ReleaseSummary {
|
|||||||
innerName: string | null
|
innerName: string | null
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Posters a poster search looks in at most: a short fragment matches thousands of random ones (1,000 took 9.9 s, 200 0.2 s; LOG.md 2026-10-16 (70)). */
|
||||||
|
const MAX_POSTERS = 200
|
||||||
const SORTS: Record<SortOrder, string> = {
|
const SORTS: Record<SortOrder, string> = {
|
||||||
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
|
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
|
||||||
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
|
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
|
||||||
@@ -92,15 +96,20 @@ class SearchService {
|
|||||||
}
|
}
|
||||||
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
|
if (params.readable) where.push('(r.readable OR r.inner_name IS NOT NULL)')
|
||||||
if (params.complete) where.push('r.complete')
|
if (params.complete) where.push('r.complete')
|
||||||
if (params.notIncomplete) where.push('r.complete IS NOT FALSE')
|
// Incomplete only when final: a release next to unread ranges may yet get its missing files (release_final).
|
||||||
|
if (params.notIncomplete) where.push('(r.complete IS NOT FALSE OR NOT release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete))')
|
||||||
if (params.unlocked) where.push('r.passworded IS NOT TRUE')
|
if (params.unlocked) where.push('r.passworded IS NOT TRUE')
|
||||||
if (params.source) {
|
if (params.source) {
|
||||||
where.push('r.source = ?')
|
where.push('r.source = ?')
|
||||||
bindings.push(params.source)
|
bindings.push(params.source)
|
||||||
}
|
}
|
||||||
|
// A poster search: the matching posters first (posters, by trigram), each one's releases then by
|
||||||
|
// releases_poster_newest_idx (below). "poster ILIKE" on releases walked all 50M rows newest first: 30+ min for a
|
||||||
|
// poster with few releases; as a filter on the posters found the planner still walked them (LOG.md 2026-10-16 (70)).
|
||||||
|
let posters: string[] | null = null
|
||||||
if (params.poster) {
|
if (params.poster) {
|
||||||
where.push('r.poster ILIKE ?')
|
const { rows: matched } = await db.rawQuery(`SELECT poster FROM posters WHERE poster ILIKE ? LIMIT ${MAX_POSTERS}`, [contains(params.poster)])
|
||||||
bindings.push(contains(params.poster))
|
posters = matched.map((m: { poster: string }) => m.poster)
|
||||||
}
|
}
|
||||||
if (params.categories?.length) {
|
if (params.categories?.length) {
|
||||||
where.push('r.category = ANY(?)')
|
where.push('r.category = ANY(?)')
|
||||||
@@ -131,13 +140,34 @@ class SearchService {
|
|||||||
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
||||||
const whereSql = where.join(' AND ')
|
const whereSql = where.join(' AND ')
|
||||||
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
||||||
|
if (posters !== null && posters.length === 0) return { page, perPage, offset, total: 0, totalCapped: false, releases: [] as ReleaseSummary[] }
|
||||||
|
// Per poster found (by first 200 characters, as posters and the index keep them; a btree entry holds at most 2,704
|
||||||
|
// bytes): its releases matching everything else, the first offset + perPage in the order asked; then merged.
|
||||||
|
const source =
|
||||||
|
posters === null
|
||||||
|
? { from: 'releases r', where: whereSql, bindings: [] as unknown[] }
|
||||||
|
: {
|
||||||
|
from: `unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
|
||||||
|
SELECT r.* FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} ORDER BY ${order} LIMIT ${offset + perPage}) r`,
|
||||||
|
where: 'true',
|
||||||
|
bindings: [posters] as unknown[],
|
||||||
|
}
|
||||||
|
const counting =
|
||||||
|
posters === null
|
||||||
|
? db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings)
|
||||||
|
: db.rawQuery(
|
||||||
|
`SELECT count(*) AS total FROM (SELECT 1 FROM unnest(?::text[]) AS p(poster) CROSS JOIN LATERAL (
|
||||||
|
SELECT 1 FROM releases r WHERE left(r.poster, 200) = p.poster AND ${whereSql} LIMIT ${countLimit + 1}) x LIMIT ${countLimit + 1}) y`,
|
||||||
|
[posters, ...bindings]
|
||||||
|
)
|
||||||
const [{ rows }, { rows: counted }] = await Promise.all([
|
const [{ rows }, { rows: counted }] = await Promise.all([
|
||||||
db.rawQuery(
|
db.rawQuery(
|
||||||
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name
|
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.files, r.bytes, r.readable, r.complete, r.category, r.settled_at, r.passworded, r.inner_name,
|
||||||
FROM releases r WHERE ${whereSql} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
release_final(r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete) AS final
|
||||||
[...bindings, perPage, offset]
|
FROM ${source.from} WHERE ${source.where} ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||||
|
[...source.bindings, ...bindings, perPage, offset]
|
||||||
),
|
),
|
||||||
db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
|
counting,
|
||||||
])
|
])
|
||||||
return {
|
return {
|
||||||
page,
|
page,
|
||||||
@@ -158,6 +188,7 @@ class SearchService {
|
|||||||
size: Number(r.bytes),
|
size: Number(r.bytes),
|
||||||
readable: r.readable,
|
readable: r.readable,
|
||||||
complete: r.complete,
|
complete: r.complete,
|
||||||
|
final: r.final,
|
||||||
category: r.category,
|
category: r.category,
|
||||||
settled: r.settled_at !== null,
|
settled: r.settled_at !== null,
|
||||||
passworded: r.passworded,
|
passworded: r.passworded,
|
||||||
@@ -170,9 +201,10 @@ class SearchService {
|
|||||||
async release(id: number) {
|
async release(id: number) {
|
||||||
const r = await db.from('releases').where('id', id).first()
|
const r = await db.from('releases').where('id', id).first()
|
||||||
if (!r) return null
|
if (!r) return null
|
||||||
const [files, { rows: inspections }] = await Promise.all([
|
const [files, { rows: inspections }, { rows: finality }] = await Promise.all([
|
||||||
NzbService.files(id),
|
NzbService.files(id),
|
||||||
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
|
db.rawQuery(`SELECT kind, result, created_at FROM release_inspections WHERE release_id = ? ORDER BY candidate_id`, [id]),
|
||||||
|
db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete]),
|
||||||
])
|
])
|
||||||
return {
|
return {
|
||||||
id: Number(r.id),
|
id: Number(r.id),
|
||||||
@@ -187,6 +219,7 @@ class SearchService {
|
|||||||
segments: Number(r.segments),
|
segments: Number(r.segments),
|
||||||
readable: r.readable,
|
readable: r.readable,
|
||||||
complete: r.complete,
|
complete: r.complete,
|
||||||
|
final: finality[0].final,
|
||||||
hasPar2: r.has_par2,
|
hasPar2: r.has_par2,
|
||||||
category: r.category,
|
category: r.category,
|
||||||
firstSeenAt: r.first_seen_at,
|
firstSeenAt: r.first_seen_at,
|
||||||
|
|||||||
@@ -18,17 +18,21 @@ const BATCH = 50_000
|
|||||||
* reads par2 sets for those). Phase 2 picks its work from settled releases.
|
* reads par2 sets for those). Phase 2 picks its work from settled releases.
|
||||||
*/
|
*/
|
||||||
class Settler {
|
class Settler {
|
||||||
async run(): Promise<number> {
|
/** Up to `batches` batches of BATCH (all there are without). */
|
||||||
|
async run(batches = Infinity): Promise<number> {
|
||||||
let settled = 0
|
let settled = 0
|
||||||
while (true) {
|
for (let batch = 0; batch < batches; batch++) {
|
||||||
// Their probe candidates become claimable (Prober.claim), the unreadable ones first.
|
// Their probe candidates become claimable (Prober.claim), the unreadable ones first.
|
||||||
const { rows } = await db.rawQuery(
|
const { rows } = await db.rawQuery(
|
||||||
`WITH settled AS (
|
`WITH settled AS (
|
||||||
UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
|
UPDATE releases SET settled_at = now(), complete = CASE WHEN file_total IS NULL THEN NULL ELSE files >= file_total END
|
||||||
WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
|
WHERE id IN (SELECT id FROM releases WHERE settled_at IS NULL AND last_seen_at < now() - make_interval(mins => ?)
|
||||||
AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
|
AND (last_seen_at < now() - make_interval(hours => ?) OR coalesce(last_posted_at, posted_at) < now() - make_interval(hours => ?)) LIMIT ?)
|
||||||
RETURNING id, readable),
|
RETURNING id, readable, files, bytes, poster),
|
||||||
marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id)
|
marked AS (UPDATE probe_candidates pc SET ready = true, unreadable = NOT s.readable FROM settled s WHERE pc.release_id = s.id),
|
||||||
|
-- Small ones may be an upload's .nfo or .nzb that came apart from it: the merger's sidecar fold looks at them.
|
||||||
|
queued AS (INSERT INTO fold_candidates (release_id) SELECT id FROM settled WHERE files <= 3 AND bytes < 1000000 AND poster IS NOT NULL
|
||||||
|
ON CONFLICT DO NOTHING)
|
||||||
SELECT count(*)::int AS n FROM settled`,
|
SELECT count(*)::int AS n FROM settled`,
|
||||||
[QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]
|
[QUIET_HISTORY_MINUTES, QUIET_HOURS, QUIET_HOURS, BATCH]
|
||||||
)
|
)
|
||||||
@@ -36,6 +40,7 @@ class Settler {
|
|||||||
settled += n
|
settled += n
|
||||||
if (n < BATCH) return settled
|
if (n < BATCH) return settled
|
||||||
}
|
}
|
||||||
|
return settled
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,91 @@
|
|||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import { decodeChunk, type ChunkFile } from '#lib/chunks/index'
|
||||||
|
|
||||||
|
/** Candidates looked at per round. */
|
||||||
|
const BATCH = 2000
|
||||||
|
/** An upload's sidecar is posted with it: its release is looked for this far from it. */
|
||||||
|
const WINDOW_MINUTES = 10
|
||||||
|
/** The files an upload carries beside its content: an index, a checksum list, a scene release's own files. */
|
||||||
|
const SIDECAR = /\.(nfo|sfv|nzb|srr|srs|txt|m3u|m3u8|diz|md5)$/i
|
||||||
|
|
||||||
|
/** The words of a name worth comparing: 4+ letters or digits. */
|
||||||
|
function words(text: string): Set<string> {
|
||||||
|
return new Set(text.toLowerCase().split(/[^a-z0-9]+/).filter((w) => w.length >= 4))
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface Sibling {
|
||||||
|
id: number
|
||||||
|
name: string
|
||||||
|
files: number
|
||||||
|
file_total: number | null
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Which of the nearby releases a sidecar belongs to: the one sharing a word of 4+ letters with the sidecar's subject
|
||||||
|
* ("Murdoch Mysteries S8D1 [00/80] - "Murdoch Mysteries S8D1.nzb"" beside "Murdoch Mysteries S8D1"), else the one
|
||||||
|
* under the same counter total ("(ALADIN) [01/34] - "bud.txt"" beside the poster's 34-file upload); none when that
|
||||||
|
* isn't one release.
|
||||||
|
*/
|
||||||
|
export function pickSibling(files: ChunkFile[], fileTotal: number | null, siblings: Sibling[]): Sibling | null {
|
||||||
|
const said = words(files.map((f) => f.subject).join(' '))
|
||||||
|
const named = siblings.filter((s) => [...words(s.name)].some((w) => said.has(w)))
|
||||||
|
if (named.length === 1) return named[0]
|
||||||
|
if (named.length > 1) return null
|
||||||
|
const counted = fileTotal ? siblings.filter((s) => s.file_total === fileTotal) : []
|
||||||
|
return counted.length === 1 ? counted[0] : null
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FoldRound {
|
||||||
|
looked: number
|
||||||
|
sidecars: number
|
||||||
|
folded: number
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The sidecar fold (REBUILD.md, junk): an .nfo, .sfv or .nzb that came apart from its upload (another name, another
|
||||||
|
* counter: "[00/80] … .nzb" beside "[01/94] … .part01.rar") was a release of its own. The settler queues small
|
||||||
|
* settled releases (fold_candidates); a candidate whose files are all sidecars joins the poster's bigger release in
|
||||||
|
* the same group posted within WINDOW_MINUTES, the one pickSibling chooses, through the merger's merge (via
|
||||||
|
* 'sidecar'). The rest leave the queue untouched.
|
||||||
|
*/
|
||||||
|
export class SidecarFolder {
|
||||||
|
constructor(private merge: (trx: any, found: Map<string, any>, key: string, via: 'sidecar') => Promise<number>) {}
|
||||||
|
|
||||||
|
async round(): Promise<FoldRound> {
|
||||||
|
const round: FoldRound = { looked: 0, sidecars: 0, folded: 0 }
|
||||||
|
const { rows: queued } = await db.rawQuery(
|
||||||
|
`SELECT r.id, r.release_key, r.group_id, r.poster, r.posted_at, r.files, r.file_total, r.readable
|
||||||
|
FROM fold_candidates f JOIN releases r ON r.id = f.release_id ORDER BY f.created_at LIMIT ?`,
|
||||||
|
[BATCH]
|
||||||
|
)
|
||||||
|
const { rows: stale } = await db.rawQuery(
|
||||||
|
`SELECT f.release_id FROM fold_candidates f LEFT JOIN releases r ON r.id = f.release_id WHERE r.id IS NULL LIMIT ?`,
|
||||||
|
[BATCH]
|
||||||
|
)
|
||||||
|
const done: number[] = stale.map((r: { release_id: string }) => Number(r.release_id))
|
||||||
|
for (const c of queued) {
|
||||||
|
round.looked++
|
||||||
|
done.push(Number(c.id))
|
||||||
|
const { rows: chunks } = await db.rawQuery(`SELECT payload FROM release_chunks WHERE release_id = ?`, [c.id])
|
||||||
|
const files: ChunkFile[] = chunks.flatMap((x: { payload: Buffer }) => decodeChunk(x.payload))
|
||||||
|
if (files.length === 0 || !files.every((f) => SIDECAR.test(f.name))) continue
|
||||||
|
round.sidecars++
|
||||||
|
if (!c.posted_at) continue
|
||||||
|
const { rows: siblings } = await db.rawQuery(
|
||||||
|
`SELECT r.id, r.release_key, r.group_id, r.files, r.file_total, r.posted_at, r.readable, r.name FROM releases r
|
||||||
|
WHERE r.posted_at BETWEEN ?::timestamptz - make_interval(mins => ?) AND ?::timestamptz + make_interval(mins => ?)
|
||||||
|
AND r.group_id = ? AND r.poster = ? AND r.id <> ? AND r.files > ?`,
|
||||||
|
[c.posted_at, WINDOW_MINUTES, c.posted_at, WINDOW_MINUTES, c.group_id, c.poster, c.id, c.files]
|
||||||
|
)
|
||||||
|
const sibling = pickSibling(files, c.file_total ? Number(c.file_total) : null, siblings.map((s: any) => ({ ...s, id: Number(s.id), files: Number(s.files), file_total: s.file_total === null ? null : Number(s.file_total) })))
|
||||||
|
if (!sibling) continue
|
||||||
|
const full = siblings.find((s: any) => Number(s.id) === sibling.id)
|
||||||
|
const merged = await db.transaction((trx) =>
|
||||||
|
this.merge(trx, new Map([[String(c.id), c], [String(full.id), full]]), `sidecar:${full.id}`, 'sidecar')
|
||||||
|
)
|
||||||
|
if (merged > 0) round.folded++
|
||||||
|
}
|
||||||
|
if (done.length > 0) await db.from('fold_candidates').whereIn('release_id', done).delete()
|
||||||
|
return round
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -14,6 +14,8 @@ export const signupValidator = vine.create({
|
|||||||
email: email().unique({ table: 'users', column: 'email' }),
|
email: email().unique({ table: 'users', column: 'email' }),
|
||||||
password: password(),
|
password: password(),
|
||||||
passwordConfirmation: password().sameAs('password'),
|
passwordConfirmation: password().sameAs('password'),
|
||||||
|
// Needed when SIGNUP is `invite` (config/access.ts).
|
||||||
|
invite: vine.string().trim().maxLength(32).optional(),
|
||||||
})
|
})
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||||
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import AccessService from '#services/access_service'
|
||||||
|
import { TIERS } from '#config/access'
|
||||||
|
|
||||||
|
/** Makes invite codes (SIGNUP=invite), optionally with a tier for the new account, e.g. a sold VIP year. */
|
||||||
|
export default class AccessInvite extends BaseCommand {
|
||||||
|
static commandName = 'access:invite'
|
||||||
|
static description = 'Makes invite codes, optionally giving a tier (e.g. --tier=vip --days=365)'
|
||||||
|
static options: CommandOptions = { startApp: true }
|
||||||
|
|
||||||
|
@flags.number({ description: 'How many codes', default: 1 })
|
||||||
|
declare count: number
|
||||||
|
|
||||||
|
@flags.string({ description: `Tier the new account gets (${Object.keys(TIERS).join(', ')})` })
|
||||||
|
declare tier?: string
|
||||||
|
|
||||||
|
@flags.number({ description: 'For how many days the tier holds (without: no end)' })
|
||||||
|
declare days?: number
|
||||||
|
|
||||||
|
@flags.string({ description: 'A note, e.g. who it is for or the payment' })
|
||||||
|
declare note?: string
|
||||||
|
|
||||||
|
async run() {
|
||||||
|
if (this.tier && !TIERS[this.tier]) {
|
||||||
|
this.logger.error(`No such tier: ${this.tier}`)
|
||||||
|
this.exitCode = 1
|
||||||
|
return
|
||||||
|
}
|
||||||
|
const codes = Array.from({ length: this.count }, () => AccessService.newKey())
|
||||||
|
await db.table('invites').multiInsert(
|
||||||
|
codes.map((code) => ({ code, tier: this.tier ?? null, tier_days: this.tier ? (this.days ?? null) : null, note: this.note ?? null }))
|
||||||
|
)
|
||||||
|
for (const code of codes) this.logger.log(code)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||||
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||||
|
import User from '#models/user'
|
||||||
|
|
||||||
|
/** Suspends a user (Newznab error 101, the catalog 403), e.g. for a shared key; --undo lifts it. */
|
||||||
|
export default class AccessSuspend extends BaseCommand {
|
||||||
|
static commandName = 'access:suspend'
|
||||||
|
static description = 'Suspends a user (or lifts it with --undo)'
|
||||||
|
static options: CommandOptions = { startApp: true }
|
||||||
|
|
||||||
|
@args.string({ description: 'The user’s email' })
|
||||||
|
declare email: string
|
||||||
|
|
||||||
|
@flags.boolean({ description: 'Lift the suspension' })
|
||||||
|
declare undo: boolean
|
||||||
|
|
||||||
|
async run() {
|
||||||
|
const user = await User.findBy('email', this.email)
|
||||||
|
if (!user) {
|
||||||
|
this.logger.error(`No user ${this.email}`)
|
||||||
|
this.exitCode = 1
|
||||||
|
return
|
||||||
|
}
|
||||||
|
user.suspended = !this.undo
|
||||||
|
await user.save()
|
||||||
|
this.logger.info(`${user.email}: ${user.suspended ? 'suspended' : 'active'}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,32 @@
|
|||||||
|
import { BaseCommand, args, flags } from '@adonisjs/core/ace'
|
||||||
|
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||||
|
import User from '#models/user'
|
||||||
|
import AccessService from '#services/access_service'
|
||||||
|
|
||||||
|
/** Gives a user a tier (a payment, by hand until one is taken on the site); a running one is extended. */
|
||||||
|
export default class AccessTier extends BaseCommand {
|
||||||
|
static commandName = 'access:tier'
|
||||||
|
static description = "Sets a user's tier, e.g. access:tier someone@example.com vip --days=365"
|
||||||
|
static options: CommandOptions = { startApp: true }
|
||||||
|
|
||||||
|
@args.string({ description: 'The user’s email' })
|
||||||
|
declare email: string
|
||||||
|
|
||||||
|
@args.string({ description: 'Tier (config/access.ts)' })
|
||||||
|
declare tier: string
|
||||||
|
|
||||||
|
@flags.number({ description: 'For how many days (without: no end); the same tier still running is extended' })
|
||||||
|
declare days?: number
|
||||||
|
|
||||||
|
async run() {
|
||||||
|
const user = await User.findBy('email', this.email)
|
||||||
|
if (!user) {
|
||||||
|
this.logger.error(`No user ${this.email}`)
|
||||||
|
this.exitCode = 1
|
||||||
|
return
|
||||||
|
}
|
||||||
|
AccessService.grantTier(user, this.tier, this.days ?? null)
|
||||||
|
await user.save()
|
||||||
|
this.logger.info(`${user.email}: ${user.tier} ${user.tierUntil ? `until ${user.tierUntil.toISO()}` : 'without end'}`)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import { Prefetcher } from '#services/prefetcher'
|
|||||||
import Planner from '#services/planner'
|
import Planner from '#services/planner'
|
||||||
import FragmentMerger from '#services/fragment_merger'
|
import FragmentMerger from '#services/fragment_merger'
|
||||||
import Settler from '#services/settler'
|
import Settler from '#services/settler'
|
||||||
|
import RangeBlocks from '#services/range_blocks'
|
||||||
import NntpPool from '#services/nntp_pool'
|
import NntpPool from '#services/nntp_pool'
|
||||||
import coordinatorConfig from '#config/coordinator'
|
import coordinatorConfig from '#config/coordinator'
|
||||||
|
|
||||||
@@ -13,6 +14,15 @@ const PLAN_EVERY_MS = 60 * 60 * 1000
|
|||||||
const MERGE_EVERY_MS = 60 * 1000
|
const MERGE_EVERY_MS = 60 * 1000
|
||||||
const MERGE_BUDGET_MS = 30 * 1000
|
const MERGE_BUDGET_MS = 30 * 1000
|
||||||
const STOP_WAIT_MS = 15 * 1000
|
const STOP_WAIT_MS = 15 * 1000
|
||||||
|
/**
|
||||||
|
* Settling in rounds of at most SETTLE_BATCHES batches every few minutes: once an hour, ~3M releases rewrote their
|
||||||
|
* rows and index entries in one go (2.65M in 30 min), the NVMe saturated and the batch stores queued on their locks
|
||||||
|
* (LOG.md 2026-10-16 (67)).
|
||||||
|
*/
|
||||||
|
const SETTLE_EVERY_MS = 5 * 60 * 1000
|
||||||
|
const SETTLE_BATCHES = 4
|
||||||
|
/** range_blocks (release_final) after ranges finish. */
|
||||||
|
const BLOCKS_EVERY_MS = 5 * 60 * 1000
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The coordinator's background work (REBUILD.md, part 1): the prefetcher,
|
* The coordinator's background work (REBUILD.md, part 1): the prefetcher,
|
||||||
@@ -56,17 +66,32 @@ export default class CoordinatorRun extends BaseCommand {
|
|||||||
await sleep(TICK_MS)
|
await sleep(TICK_MS)
|
||||||
}
|
}
|
||||||
})()
|
})()
|
||||||
|
// range_blocks in a loop of its own: a settling or merging round doesn't hold release_final back.
|
||||||
|
const blocks = (async () => {
|
||||||
|
while (main && !stopping) {
|
||||||
|
try {
|
||||||
|
await RangeBlocks.rebuild()
|
||||||
|
} catch (error: any) {
|
||||||
|
this.logger.error(`Range blocks failed: ${error.message}`)
|
||||||
|
}
|
||||||
|
for (let waited = 0; waited < BLOCKS_EVERY_MS && !stopping; waited += TICK_MS) await sleep(TICK_MS)
|
||||||
|
}
|
||||||
|
})()
|
||||||
// History for new groups hourly; the tail every few minutes.
|
// History for new groups hourly; the tail every few minutes.
|
||||||
let plannedAt = 0
|
let plannedAt = 0
|
||||||
let tailedAt = 0
|
let tailedAt = 0
|
||||||
let mergedAt = 0
|
let mergedAt = 0
|
||||||
|
let settledAt = 0
|
||||||
while (!stopping) {
|
while (!stopping) {
|
||||||
try {
|
try {
|
||||||
if (main && Date.now() - plannedAt >= PLAN_EVERY_MS) {
|
if (main && Date.now() - plannedAt >= PLAN_EVERY_MS) {
|
||||||
plannedAt = Date.now()
|
plannedAt = Date.now()
|
||||||
const { groups, ranges } = await Planner.planAll()
|
const { groups, ranges } = await Planner.planAll()
|
||||||
if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
|
if (ranges > 0) this.logger.info(`Planned ${ranges} ranges over ${groups} groups.`)
|
||||||
const settled = await Settler.run()
|
}
|
||||||
|
if (main && Date.now() - settledAt >= SETTLE_EVERY_MS) {
|
||||||
|
settledAt = Date.now()
|
||||||
|
const settled = await Settler.run(SETTLE_BATCHES)
|
||||||
if (settled > 0) this.logger.info(`Settled ${settled} releases.`)
|
if (settled > 0) this.logger.info(`Settled ${settled} releases.`)
|
||||||
}
|
}
|
||||||
if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
|
if (main && Date.now() - tailedAt >= coordinatorConfig.tail.everySeconds * 1000) {
|
||||||
@@ -98,7 +123,7 @@ export default class CoordinatorRun extends BaseCommand {
|
|||||||
}
|
}
|
||||||
await sleep(TICK_MS)
|
await sleep(TICK_MS)
|
||||||
}
|
}
|
||||||
await prefetching
|
await Promise.all([prefetching, blocks])
|
||||||
// The chunks in flight, for at most STOP_WAIT_MS (a connection the server holds back can take minutes); then closed.
|
// The chunks in flight, for at most STOP_WAIT_MS (a connection the server holds back can take minutes); then closed.
|
||||||
const waitUntil = Date.now() + STOP_WAIT_MS
|
const waitUntil = Date.now() + STOP_WAIT_MS
|
||||||
while (prefetcher.active > 0 && Date.now() < waitUntil) await sleep(200)
|
while (prefetcher.active > 0 && Date.now() < waitUntil) await sleep(200)
|
||||||
|
|||||||
@@ -0,0 +1,31 @@
|
|||||||
|
import env from '#start/env'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Who may use the catalog and the Newznab API.
|
||||||
|
*
|
||||||
|
* `open` (the default): as before, one key for Newznab (NEWZNAB_API_KEY, or
|
||||||
|
* none) and an open web catalog, no limits. `accounts`: every request names a
|
||||||
|
* user (a Newznab `apikey`, or the web interface's token), whose tier sets what
|
||||||
|
* they may do per UTC day and how far back they see.
|
||||||
|
*/
|
||||||
|
export const ACCESS = env.get('ACCESS', 'open')
|
||||||
|
|
||||||
|
/** Who may sign up: anyone, with an unused invite code, or no one. With accounts, invites unless set. */
|
||||||
|
export const SIGNUP = env.get('SIGNUP', ACCESS === 'accounts' ? 'invite' : 'open')
|
||||||
|
|
||||||
|
export interface TierLimits {
|
||||||
|
/** Searches and release details per day (Newznab t=search & co., the catalog); null: no limit. */
|
||||||
|
apiHits: number | null
|
||||||
|
/** NZBs per day; null: no limit. */
|
||||||
|
grabs: number | null
|
||||||
|
/** How many days back releases are found and fetched; null: everything. */
|
||||||
|
retentionDays: number | null
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A tier that runs out (users.tier_until) falls back to `free`. */
|
||||||
|
export const TIERS: Record<string, TierLimits> = {
|
||||||
|
free: { apiHits: 100, grabs: 10, retentionDays: 365 },
|
||||||
|
vip: { apiHits: 10_000, grabs: 1_000, retentionDays: null },
|
||||||
|
// Staff and friends.
|
||||||
|
unlimited: { apiHits: null, grabs: null, retentionDays: null },
|
||||||
|
}
|
||||||
@@ -23,8 +23,11 @@ export default {
|
|||||||
everySeconds: env.get('TAIL_EVERY_SECONDS', 120),
|
everySeconds: env.get('TAIL_EVERY_SECONDS', 120),
|
||||||
minArticles: env.get('TAIL_MIN_ARTICLES', 100_000),
|
minArticles: env.get('TAIL_MIN_ARTICLES', 100_000),
|
||||||
maxDelayMinutes: env.get('TAIL_MAX_DELAY_MINUTES', 15),
|
maxDelayMinutes: env.get('TAIL_MAX_DELAY_MINUTES', 15),
|
||||||
/** Before the history's 0 (oldest) … 1000 (newest). */
|
/**
|
||||||
priority: -1,
|
* Before everything: the history's 0 (oldest) … 1000 (newest), the stored ranges' -2 and ranges raised to -1 (the
|
||||||
|
* short gaps between done blocks went before the tail at -1 and it fell ~4 h behind; LOG.md 2026-10-16 (70)).
|
||||||
|
*/
|
||||||
|
priority: -3,
|
||||||
},
|
},
|
||||||
/** A leased range without a heartbeat for this long goes back to the queue. */
|
/** A leased range without a heartbeat for this long goes back to the queue. */
|
||||||
leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
|
leaseTtlSeconds: env.get('LEASE_TTL_SECONDS', 15 * 60),
|
||||||
|
|||||||
@@ -0,0 +1,49 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Accounts with limits (config/access.ts): a Newznab key and a tier per user,
|
||||||
|
* invites, and what each user used per day.
|
||||||
|
*/
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
async up() {
|
||||||
|
this.schema.alterTable('users', (table) => {
|
||||||
|
// Existing users get a key of their own (a volatile default is evaluated per row).
|
||||||
|
table.string('api_key', 32).notNullable().unique().defaultTo(this.raw(`md5(random()::text || clock_timestamp()::text)`))
|
||||||
|
table.string('tier', 16).notNullable().defaultTo('free')
|
||||||
|
// Null: the tier doesn't run out.
|
||||||
|
table.timestamp('tier_until', { useTz: true }).nullable()
|
||||||
|
table.boolean('suspended').notNullable().defaultTo(false)
|
||||||
|
})
|
||||||
|
|
||||||
|
this.schema.createTable('invites', (table) => {
|
||||||
|
table.string('code', 32).primary()
|
||||||
|
// What the invite gives (e.g. a sold VIP invite); null: the free tier.
|
||||||
|
table.string('tier', 16).nullable()
|
||||||
|
table.integer('tier_days').nullable()
|
||||||
|
table.string('note').nullable()
|
||||||
|
table.integer('used_by').nullable().references('id').inTable('users').onDelete('SET NULL')
|
||||||
|
table.timestamp('used_at', { useTz: true }).nullable()
|
||||||
|
table.timestamp('created_at', { useTz: true }).notNullable().defaultTo(this.now())
|
||||||
|
})
|
||||||
|
|
||||||
|
this.schema.createTable('usage', (table) => {
|
||||||
|
table.integer('user_id').notNullable().references('id').inTable('users').onDelete('CASCADE')
|
||||||
|
// A UTC day.
|
||||||
|
table.date('day').notNullable()
|
||||||
|
table.integer('api_hits').notNullable().defaultTo(0)
|
||||||
|
table.integer('grabs').notNullable().defaultTo(0)
|
||||||
|
table.primary(['user_id', 'day'])
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {
|
||||||
|
this.schema.dropTable('usage')
|
||||||
|
this.schema.dropTable('invites')
|
||||||
|
this.schema.alterTable('users', (table) => {
|
||||||
|
table.dropColumn('api_key')
|
||||||
|
table.dropColumn('tier')
|
||||||
|
table.dropColumn('tier_until')
|
||||||
|
table.dropColumn('suspended')
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Small releases the settler just settled (at most 3 files, under 1 MB), queued for the merger's sidecar fold
|
||||||
|
* (app/services/sidecar_folder.ts): an .nfo, .sfv or .nzb that came apart from its upload (another counter, another
|
||||||
|
* name) joins it. A queue, so the fold needn't scan the releases (LOG.md 2026-10-15 (57)).
|
||||||
|
*/
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
async up() {
|
||||||
|
this.schema.raw(`
|
||||||
|
CREATE TABLE fold_candidates (
|
||||||
|
release_id bigint PRIMARY KEY,
|
||||||
|
created_at timestamptz NOT NULL DEFAULT now()
|
||||||
|
)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {
|
||||||
|
this.schema.raw(`DROP TABLE IF EXISTS fold_candidates`)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* fragments, with its indexes, back in the catalog's tablespace (default_tablespace: nvme on homelab), out of
|
||||||
|
* CHUNK_TABLESPACE where 3200 put it: it isn't read by key only. The fragment merger walks its file_key index and
|
||||||
|
* reads each row, written and deleted all the time; on the HDD one round of its 30 s budget took 12+ minutes, and the
|
||||||
|
* coordinator's settling and tail planning waited behind it (LOG.md 2026-10-15 (57)). 13 GB. Homelab was moved by
|
||||||
|
* hand first, with the writers stopped; a no-op where it's there already, or without a default tablespace (tests).
|
||||||
|
*/
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
async up() {
|
||||||
|
this.defer(async (db) => {
|
||||||
|
const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
|
||||||
|
const target = setting[0].target
|
||||||
|
if (!target) return
|
||||||
|
const { rows } = await db.rawQuery(
|
||||||
|
`SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
||||||
|
WHERE (c.relname = 'fragments' OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = 'fragments'::regclass))
|
||||||
|
AND coalesce(t.spcname, 'pg_default') <> ?`,
|
||||||
|
[target]
|
||||||
|
)
|
||||||
|
for (const { relname, relkind } of rows) {
|
||||||
|
await db.rawQuery(`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${target}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {}
|
||||||
|
}
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Where a worker's incremental rounds go on after a restart. The merger kept its "since" in memory only: each deploy
|
||||||
|
* or pause started it from 1970, a first round over every par2 set and inner stem ever found (25+ min with par2_files
|
||||||
|
* on the HDD) before the sidecar fold's turn (LOG.md 2026-10-15 (58)).
|
||||||
|
*/
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
async up() {
|
||||||
|
this.schema.raw(`
|
||||||
|
CREATE TABLE worker_cursors (
|
||||||
|
name text PRIMARY KEY,
|
||||||
|
at timestamptz NOT NULL
|
||||||
|
)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {
|
||||||
|
this.schema.raw(`DROP TABLE IF EXISTS worker_cursors`)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,29 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* par2_files, with its indexes, back in the catalog's tablespace (default_tablespace: nvme on homelab), out of
|
||||||
|
* CHUNK_TABLESPACE where 3200 put it: the merger matches every new file hash against it and reads each set's rows, a
|
||||||
|
* random read per row on the HDD (~1,090 sets/min, a full round ~28 h; LOG.md 2026-10-15 (58)). Homelab rebuilt it
|
||||||
|
* there by hand without the complete and judged sets' rows (app/services/par2_pruner.ts; LOG.md 2026-10-16 (60)); a
|
||||||
|
* no-op where it's there already, or without a default tablespace (tests).
|
||||||
|
*/
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
async up() {
|
||||||
|
this.defer(async (db) => {
|
||||||
|
const { rows: setting } = await db.rawQuery(`SELECT current_setting('default_tablespace') AS target`)
|
||||||
|
const target = setting[0].target
|
||||||
|
if (!target) return
|
||||||
|
const { rows } = await db.rawQuery(
|
||||||
|
`SELECT c.relname, c.relkind FROM pg_class c LEFT JOIN pg_tablespace t ON t.oid = c.reltablespace
|
||||||
|
WHERE (c.relname = 'par2_files' OR c.oid IN (SELECT indexrelid FROM pg_index WHERE indrelid = 'par2_files'::regclass))
|
||||||
|
AND coalesce(t.spcname, 'pg_default') <> ?`,
|
||||||
|
[target]
|
||||||
|
)
|
||||||
|
for (const { relname, relkind } of rows) {
|
||||||
|
await db.rawQuery(`ALTER ${relkind === 'i' ? 'INDEX' : 'TABLE'} ${relname} SET TABLESPACE ${target}`)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A rule set version with droppedPosters (lib/rules/rule_set.ts): the newest set plus the CelebMovie family, whose
|
||||||
|
* scraped celebrity pictures workers drop at the header pass from now on (LOG.md 2026-10-16 (65)). A new version, so the
|
||||||
|
* ranges record which ran with it. Nothing where there's no rule set yet (the default gets it) or the newest has it.
|
||||||
|
*/
|
||||||
|
const POSTERS = ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes']
|
||||||
|
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
async up() {
|
||||||
|
this.defer(async (db) => {
|
||||||
|
const latest = await db.from('rule_sets').orderBy('version', 'desc').first()
|
||||||
|
if (!latest || 'droppedPosters' in latest.rules) return
|
||||||
|
await db.table('rule_sets').insert({
|
||||||
|
version: Number(latest.version) + 1,
|
||||||
|
rules: JSON.stringify({ ...latest.rules, droppedPosters: POSTERS }),
|
||||||
|
note: 'dropped posters: the CelebMovie family (2026-10-16)',
|
||||||
|
})
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {}
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Final completeness by time window (LOG.md 2026-10-16 (67)). A release settles once quiet, but history is read
|
||||||
|
* newest first and in islands: one next to an unread range was "incomplete" twice as often (52.5% against 25.7%),
|
||||||
|
* its missing files only not read yet. An upload is posted within hours, so a release is final (its complete verdict
|
||||||
|
* definite) once done ranges cover its posting times and FINAL_MARGIN either side.
|
||||||
|
* - ranges.posted_low/high: the posting time span the workers report per batch (postedSpan: without the earliest and
|
||||||
|
* latest 1%), the min and max over the range's batches.
|
||||||
|
* - range_blocks: runs of consecutive done ranges per group with their span (RangeBlocks rebuilds it); at_group_start
|
||||||
|
* when the run starts at the group's first range (nothing older to wait for).
|
||||||
|
* - release_final(): whether one block holds the release's articles and reaches FINAL_MARGIN past its posting times.
|
||||||
|
* Computed when asked, so no release row is written (50M updates would double the table on the NVMe).
|
||||||
|
*/
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
async up() {
|
||||||
|
this.schema.raw(`ALTER TABLE ranges ADD COLUMN IF NOT EXISTS posted_low timestamptz, ADD COLUMN IF NOT EXISTS posted_high timestamptz`)
|
||||||
|
this.schema.raw(`
|
||||||
|
CREATE TABLE range_blocks (
|
||||||
|
group_id integer NOT NULL,
|
||||||
|
start_id bigint NOT NULL,
|
||||||
|
end_id bigint NOT NULL,
|
||||||
|
posted_low timestamptz,
|
||||||
|
posted_high timestamptz,
|
||||||
|
at_group_start boolean NOT NULL,
|
||||||
|
PRIMARY KEY (group_id, start_id)
|
||||||
|
)`)
|
||||||
|
// FINAL_MARGIN: 6 hours.
|
||||||
|
this.schema.raw(`
|
||||||
|
CREATE FUNCTION release_final(g integer, first_article bigint, last_article bigint, posted timestamptz, last_posted timestamptz)
|
||||||
|
RETURNS boolean LANGUAGE sql STABLE AS $$
|
||||||
|
SELECT EXISTS (
|
||||||
|
SELECT 1 FROM range_blocks b
|
||||||
|
WHERE b.group_id = g AND b.start_id <= first_article AND b.end_id >= last_article
|
||||||
|
AND (b.at_group_start OR b.posted_low <= posted - interval '6 hours')
|
||||||
|
AND b.posted_high >= coalesce(last_posted, posted) + interval '6 hours')
|
||||||
|
$$`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {
|
||||||
|
this.schema.raw(`DROP FUNCTION IF EXISTS release_final(integer, bigint, bigint, timestamptz, timestamptz)`)
|
||||||
|
this.schema.raw(`DROP TABLE IF EXISTS range_blocks`)
|
||||||
|
this.schema.raw(`ALTER TABLE ranges DROP COLUMN IF EXISTS posted_low, DROP COLUMN IF EXISTS posted_high`)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Poster search (LOG.md 2026-10-16 (70)): "poster ILIKE '%…%'" on releases had no index, so it walked all 50M releases
|
||||||
|
* newest first; a poster with few releases took 30+ minutes. posters holds each poster once (3M, many random per file
|
||||||
|
* of an obfuscated upload) with a trigram index; releases_poster_newest_idx gives a poster's releases newest first.
|
||||||
|
* Both by the poster's first 200 characters: a spam poster of 3,024 bytes is over a btree entry's 2,704. The batch store
|
||||||
|
* adds new posters. Homelab built the releases index by hand first (concurrently); a no-op there.
|
||||||
|
*/
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
static disableTransactions = true
|
||||||
|
|
||||||
|
async up() {
|
||||||
|
this.schema.raw(`CREATE TABLE IF NOT EXISTS posters (poster text PRIMARY KEY)`)
|
||||||
|
this.schema.raw(`INSERT INTO posters (poster) SELECT DISTINCT left(poster, 200) FROM releases WHERE poster IS NOT NULL ON CONFLICT DO NOTHING`)
|
||||||
|
this.schema.raw(`CREATE INDEX IF NOT EXISTS posters_trgm ON posters USING gin (poster gin_trgm_ops)`)
|
||||||
|
this.schema.raw(`CREATE INDEX CONCURRENTLY IF NOT EXISTS releases_poster_newest_idx ON releases (left(poster, 200), posted_at DESC NULLS LAST, id DESC)`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {
|
||||||
|
this.schema.raw(`DROP INDEX CONCURRENTLY IF EXISTS releases_poster_newest_idx`)
|
||||||
|
this.schema.raw(`DROP TABLE IF EXISTS posters`)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,23 @@
|
|||||||
|
import { BaseSchema } from '@adonisjs/lucid/schema'
|
||||||
|
|
||||||
|
/**
|
||||||
|
* release_final() with the release's complete flag: a release with every file its counter announced is final as it is
|
||||||
|
* (nothing it lacks can come); the time window only decides for the incomplete and those without a counter (LOG.md
|
||||||
|
* 2026-10-16 (70): a complete episode posted an hour before was "pending" for six hours more).
|
||||||
|
*/
|
||||||
|
export default class extends BaseSchema {
|
||||||
|
async up() {
|
||||||
|
// Beside the five-argument one, which the services still running call until the deploy restarts them.
|
||||||
|
this.schema.raw(`
|
||||||
|
CREATE FUNCTION release_final(g integer, first_article bigint, last_article bigint, posted timestamptz, last_posted timestamptz, complete boolean)
|
||||||
|
RETURNS boolean LANGUAGE sql STABLE AS $$
|
||||||
|
SELECT coalesce(complete, false) OR EXISTS (
|
||||||
|
SELECT 1 FROM range_blocks b
|
||||||
|
WHERE b.group_id = g AND b.start_id <= first_article AND b.end_id >= last_article
|
||||||
|
AND (b.at_group_start OR b.posted_low <= posted - interval '6 hours')
|
||||||
|
AND b.posted_high >= coalesce(last_posted, posted) + interval '6 hours')
|
||||||
|
$$`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async down() {}
|
||||||
|
}
|
||||||
+41
-1
@@ -87,6 +87,25 @@ export class GroupSchema extends BaseModel {
|
|||||||
declare tail: boolean
|
declare tail: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class InviteSchema extends BaseModel {
|
||||||
|
static $columns = ['code', 'createdAt', 'note', 'tier', 'tierDays', 'usedAt', 'usedBy'] as const
|
||||||
|
$columns = InviteSchema.$columns
|
||||||
|
@column({ isPrimary: true })
|
||||||
|
declare code: string
|
||||||
|
@column.dateTime({ autoCreate: true })
|
||||||
|
declare createdAt: DateTime
|
||||||
|
@column()
|
||||||
|
declare note: string | null
|
||||||
|
@column()
|
||||||
|
declare tier: string | null
|
||||||
|
@column()
|
||||||
|
declare tierDays: number | null
|
||||||
|
@column.dateTime()
|
||||||
|
declare usedAt: DateTime | null
|
||||||
|
@column()
|
||||||
|
declare usedBy: number | null
|
||||||
|
}
|
||||||
|
|
||||||
export class NodeAccessTokenSchema extends BaseModel {
|
export class NodeAccessTokenSchema extends BaseModel {
|
||||||
static $columns = ['abilities', 'createdAt', 'expiresAt', 'hash', 'id', 'lastUsedAt', 'name', 'tokenableId', 'type', 'updatedAt'] as const
|
static $columns = ['abilities', 'createdAt', 'expiresAt', 'hash', 'id', 'lastUsedAt', 'name', 'tokenableId', 'type', 'updatedAt'] as const
|
||||||
$columns = NodeAccessTokenSchema.$columns
|
$columns = NodeAccessTokenSchema.$columns
|
||||||
@@ -399,9 +418,24 @@ export class RuleSetSchema extends BaseModel {
|
|||||||
declare version: number
|
declare version: number
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export class UsageSchema extends BaseModel {
|
||||||
|
static $columns = ['apiHits', 'day', 'grabs', 'userId'] as const
|
||||||
|
$columns = UsageSchema.$columns
|
||||||
|
@column()
|
||||||
|
declare apiHits: number
|
||||||
|
@column.date()
|
||||||
|
declare day: DateTime
|
||||||
|
@column()
|
||||||
|
declare grabs: number
|
||||||
|
@column({ isPrimary: true })
|
||||||
|
declare userId: number
|
||||||
|
}
|
||||||
|
|
||||||
export class UserSchema extends BaseModel {
|
export class UserSchema extends BaseModel {
|
||||||
static $columns = ['createdAt', 'email', 'fullName', 'id', 'password', 'updatedAt'] as const
|
static $columns = ['apiKey', 'createdAt', 'email', 'fullName', 'id', 'password', 'suspended', 'tier', 'tierUntil', 'updatedAt'] as const
|
||||||
$columns = UserSchema.$columns
|
$columns = UserSchema.$columns
|
||||||
|
@column()
|
||||||
|
declare apiKey: string
|
||||||
@column.dateTime({ autoCreate: true })
|
@column.dateTime({ autoCreate: true })
|
||||||
declare createdAt: DateTime
|
declare createdAt: DateTime
|
||||||
@column()
|
@column()
|
||||||
@@ -412,6 +446,12 @@ export class UserSchema extends BaseModel {
|
|||||||
declare id: number
|
declare id: number
|
||||||
@column({ serializeAs: null })
|
@column({ serializeAs: null })
|
||||||
declare password: string
|
declare password: string
|
||||||
|
@column()
|
||||||
|
declare suspended: boolean
|
||||||
|
@column()
|
||||||
|
declare tier: string
|
||||||
|
@column.dateTime()
|
||||||
|
declare tierUntil: DateTime | null
|
||||||
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
@column.dateTime({ autoCreate: true, autoUpdate: true })
|
||||||
declare updatedAt: DateTime | null
|
declare updatedAt: DateTime | null
|
||||||
}
|
}
|
||||||
+14
-2
@@ -46,19 +46,31 @@ services:
|
|||||||
# waited on it (WALWrite, 56–70 ms a write with the HDD 100% busy; LOG.md 2026-10-13 (43)).
|
# waited on it (WALWrite, 56–70 ms a write with the HDD 100% busy; LOG.md 2026-10-13 (43)).
|
||||||
- ${PG_WAL_DIR:-/var/lib/usenet-rebuild-wal}:/var/lib/postgresql/wal
|
- ${PG_WAL_DIR:-/var/lib/usenet-rebuild-wal}:/var/lib/postgresql/wal
|
||||||
# synchronous_commit = off is set with ALTER SYSTEM (postgresql.auto.conf in the data dir).
|
# synchronous_commit = off is set with ALTER SYSTEM (postgresql.auto.conf in the data dir).
|
||||||
|
# Memory for 62 GiB of RAM shared with other services (LOG.md 2026-10-16 (69)): the catalog's hot part in
|
||||||
|
# shared_buffers, the planner told the OS caches the rest; random_page_cost for the NVMe (the HDD tablespace
|
||||||
|
# pg_default keeps 4: ALTER TABLESPACE … SET (random_page_cost = 4)).
|
||||||
command:
|
command:
|
||||||
- postgres
|
- postgres
|
||||||
- -c
|
- -c
|
||||||
- shared_preload_libraries=pg_stat_statements
|
- shared_preload_libraries=pg_stat_statements
|
||||||
- -c
|
- -c
|
||||||
- shared_buffers=${PG_SHARED_BUFFERS:-2GB}
|
- shared_buffers=${PG_SHARED_BUFFERS:-8GB}
|
||||||
|
- -c
|
||||||
|
- effective_cache_size=${PG_EFFECTIVE_CACHE_SIZE:-32GB}
|
||||||
|
- -c
|
||||||
|
- random_page_cost=1.1
|
||||||
|
- -c
|
||||||
|
- work_mem=16MB
|
||||||
|
- -c
|
||||||
|
- maintenance_work_mem=1GB
|
||||||
- -c
|
- -c
|
||||||
- max_wal_size=8GB
|
- max_wal_size=8GB
|
||||||
- -c
|
- -c
|
||||||
- checkpoint_timeout=15min
|
- checkpoint_timeout=15min
|
||||||
- -c
|
- -c
|
||||||
- wal_buffers=64MB
|
- wal_buffers=64MB
|
||||||
shm_size: 1g
|
# Parallel hash joins' shared memory: 1g failed with "could not resize shared memory segment" (LOG.md 2026-10-16 (61)).
|
||||||
|
shm_size: 2g
|
||||||
stop_grace_period: 120s
|
stop_grace_period: 120s
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ['CMD-SHELL', 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"']
|
test: ['CMD-SHELL', 'pg_isready -U "$$POSTGRES_USER" -d "$$POSTGRES_DB"']
|
||||||
|
|||||||
+8
-2
@@ -45,8 +45,14 @@ docker compose run --rm --no-deps migrate
|
|||||||
# stream keeps the node API from exiting.
|
# stream keeps the node API from exiting.
|
||||||
echo "== recreating app services"
|
echo "== recreating app services"
|
||||||
docker compose stop edge
|
docker compose stop edge
|
||||||
docker compose up -d --no-deps node-api web coordinator prober merger judge spotter spot-inspector
|
if [ -e "$CACHE/paused" ]; then
|
||||||
docker compose up -d --no-deps edge
|
# Paused for the night (~/bin/usenet-rebuild-pause.sh): the workers stay stopped, the resume starts the new build.
|
||||||
|
echo "== paused: coordinator, prober, merger, judge, spotter, spot-inspector and edge left stopped"
|
||||||
|
docker compose up -d --no-deps node-api web
|
||||||
|
else
|
||||||
|
docker compose up -d --no-deps node-api web coordinator prober merger judge spotter spot-inspector
|
||||||
|
docker compose up -d --no-deps edge
|
||||||
|
fi
|
||||||
echo "$SHA $(date -Is)" >> "$ROOT/deployed.log"
|
echo "$SHA $(date -Is)" >> "$ROOT/deployed.log"
|
||||||
# The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor).
|
# The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor).
|
||||||
docker image prune -f --filter label=com.docker.compose.project=usenet-rebuild >/dev/null
|
docker image prune -f --filter label=com.docker.compose.project=usenet-rebuild >/dev/null
|
||||||
|
|||||||
Executable
+12
@@ -0,0 +1,12 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Installed as ~/bin/usenet-rebuild-pause.sh (by hand). Nightly (homelab crontab, 23:45): voyager 2's processing paused for the night; usenet-rebuild-resume.sh (07:45)
|
||||||
|
# starts it again. Postgres, web and node-api stay up. Takes the deploy lock, so it never runs during a CI deploy, and
|
||||||
|
# leaves .cache/paused, so a deploy in the night doesn't start the workers again.
|
||||||
|
cd /mnt/bulk/usenet-rebuild || exit 1
|
||||||
|
exec 9>.cache/deploy.lock
|
||||||
|
flock 9
|
||||||
|
touch .cache/paused
|
||||||
|
date "+%F %T pausing"
|
||||||
|
docker compose stop edge
|
||||||
|
docker compose stop coordinator prober merger judge spotter spot-inspector
|
||||||
|
docker compose ps --format "{{.Service}} {{.Status}}"
|
||||||
Executable
+11
@@ -0,0 +1,11 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Installed as ~/bin/usenet-rebuild-resume.sh (by hand). Nightly (homelab crontab, 07:45): voyager 2's processing back after usenet-rebuild-pause.sh (23:45). Takes the
|
||||||
|
# deploy lock, like the pause.
|
||||||
|
cd /mnt/bulk/usenet-rebuild || exit 1
|
||||||
|
exec 9>.cache/deploy.lock
|
||||||
|
flock 9
|
||||||
|
rm -f .cache/paused
|
||||||
|
date "+%F %T resuming"
|
||||||
|
docker compose up -d --no-deps coordinator prober merger judge spotter spot-inspector
|
||||||
|
docker compose up -d --no-deps edge
|
||||||
|
docker compose ps --format "{{.Service}} {{.Status}}"
|
||||||
+1
-1
@@ -52,7 +52,7 @@ const release = (r: Item, b: number) => {
|
|||||||
}
|
}
|
||||||
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
const ts: Record<string, Item[]> = { b: [], r: [], s: [], c: [], n: [], f: [] }
|
||||||
batches.forEach((batch, b) => {
|
batches.forEach((batch, b) => {
|
||||||
ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers })
|
ts.b.push({ b, from: batch.from, next: batch.next, headers: batch.headers, postedLow: batch.postedLow, postedHigh: batch.postedHigh })
|
||||||
for (const r of batch.releases) ts.r.push(release({ upload: r.upload, key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, postedLast: r.postedLast, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
|
for (const r of batch.releases) ts.r.push(release({ upload: r.upload, key: r.key, name: r.name, source: r.source, fileTotal: r.fileTotal, poster: r.poster, groups: r.groups, postedAt: r.postedAt, postedLast: r.postedLast, files: r.files, segments: r.segments, bytes: r.bytes, firstArticle: r.firstArticle, lastArticle: r.lastArticle, readable: r.readable, hasPar2: r.hasPar2, payload: r.payload }, b))
|
||||||
for (const s of batch.samples) ts.s.push({ b, key: s.key, upload: s.upload, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
|
for (const s of batch.samples) ts.s.push({ b, key: s.key, upload: s.upload, rule: s.rule, ruleKey: s.ruleKey, segments: s.segments })
|
||||||
for (const c of batch.candidates) ts.c.push({ b, upload: c.upload, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
|
for (const c of batch.candidates) ts.c.push({ b, upload: c.upload, releaseKey: c.releaseKey, kind: c.kind, messageId: c.messageId, file: c.file })
|
||||||
|
|||||||
@@ -64,6 +64,8 @@ struct RulesJson {
|
|||||||
sample_percent: Option<f64>,
|
sample_percent: Option<f64>,
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
drop_executables: Option<bool>,
|
drop_executables: Option<bool>,
|
||||||
|
#[serde(default)]
|
||||||
|
dropped_posters: Option<Vec<String>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RulesJson {
|
impl RulesJson {
|
||||||
@@ -83,6 +85,9 @@ impl RulesJson {
|
|||||||
if let Some(e) = self.drop_executables {
|
if let Some(e) = self.drop_executables {
|
||||||
rules.drop_executables = e;
|
rules.drop_executables = e;
|
||||||
}
|
}
|
||||||
|
if let Some(posters) = self.dropped_posters {
|
||||||
|
rules.dropped_posters = posters;
|
||||||
|
}
|
||||||
rules
|
rules
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+34
-1
@@ -78,6 +78,9 @@ pub struct Batch {
|
|||||||
pub from: i64,
|
pub from: i64,
|
||||||
pub next: i64,
|
pub next: i64,
|
||||||
pub headers: u64,
|
pub headers: u64,
|
||||||
|
/// postedLow / postedHigh: the batch's posting times without the earliest and latest 1% (postedSpan).
|
||||||
|
pub posted_low: Option<i64>,
|
||||||
|
pub posted_high: Option<i64>,
|
||||||
pub releases: Vec<ReleaseChunk>,
|
pub releases: Vec<ReleaseChunk>,
|
||||||
pub samples: Vec<(ReleaseChunk, &'static str, String)>,
|
pub samples: Vec<(ReleaseChunk, &'static str, String)>,
|
||||||
pub candidates: Vec<Candidate>,
|
pub candidates: Vec<Candidate>,
|
||||||
@@ -277,6 +280,16 @@ fn first_track(text: &str) -> Option<usize> {
|
|||||||
None
|
None
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// postedSpan: posting times without the earliest and latest 1% (floor(n / 100) at each end).
|
||||||
|
pub fn posted_span(mut dates: Vec<i64>) -> (Option<i64>, Option<i64>) {
|
||||||
|
if dates.is_empty() {
|
||||||
|
return (None, None);
|
||||||
|
}
|
||||||
|
dates.sort_unstable();
|
||||||
|
let k = dates.len() / 100;
|
||||||
|
(Some(dates[k]), Some(dates[dates.len() - 1 - k]))
|
||||||
|
}
|
||||||
|
|
||||||
/// nameBesideQuotes (lib/headers/subject.ts): a file name elsewhere when the quoted text has no extension.
|
/// nameBesideQuotes (lib/headers/subject.ts): a file name elsewhere when the quoted text has no extension.
|
||||||
pub fn name_beside_quotes(quoted: &str, before_counter: &str) -> Option<String> {
|
pub fn name_beside_quotes(quoted: &str, before_counter: &str) -> Option<String> {
|
||||||
if has_extension(quoted) {
|
if has_extension(quoted) {
|
||||||
@@ -633,6 +646,8 @@ pub struct RangeIndexer {
|
|||||||
count_order: Vec<(&'static str, String)>,
|
count_order: Vec<(&'static str, String)>,
|
||||||
completed: FxHashMap<String, Vec<Vec<Seg>>>,
|
completed: FxHashMap<String, Vec<Vec<Seg>>>,
|
||||||
completed_order: Vec<String>,
|
completed_order: Vec<String>,
|
||||||
|
/// Posting times of this batch's segments (posted_span at take).
|
||||||
|
dates: Vec<i64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl RangeIndexer {
|
impl RangeIndexer {
|
||||||
@@ -653,6 +668,7 @@ impl RangeIndexer {
|
|||||||
count_order: Vec::new(),
|
count_order: Vec::new(),
|
||||||
completed: FxHashMap::default(),
|
completed: FxHashMap::default(),
|
||||||
completed_order: Vec::new(),
|
completed_order: Vec::new(),
|
||||||
|
dates: Vec::new(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -683,6 +699,7 @@ impl RangeIndexer {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
self.batch.headers += headers;
|
self.batch.headers += headers;
|
||||||
|
self.dates.extend(segments.iter().filter_map(|s| s.posted_at));
|
||||||
for owner in crossposts {
|
for owner in crossposts {
|
||||||
self.count("crosspost", owner, 1, 0);
|
self.count("crosspost", owner, 1, 0);
|
||||||
}
|
}
|
||||||
@@ -725,6 +742,7 @@ impl RangeIndexer {
|
|||||||
self.scanner.files.clear();
|
self.scanner.files.clear();
|
||||||
let next = self.batch.next;
|
let next = self.batch.next;
|
||||||
let mut out = std::mem::replace(&mut self.batch, Batch { from: next, next, ..Default::default() });
|
let mut out = std::mem::replace(&mut self.batch, Batch { from: next, next, ..Default::default() });
|
||||||
|
(out.posted_low, out.posted_high) = posted_span(std::mem::take(&mut self.dates));
|
||||||
for id in self.count_order.drain(..) {
|
for id in self.count_order.drain(..) {
|
||||||
let (articles, files) = self.counts.remove(&id).unwrap();
|
let (articles, files) = self.counts.remove(&id).unwrap();
|
||||||
out.counts.push((id.0, id.1, articles, files));
|
out.counts.push((id.0, id.1, articles, files));
|
||||||
@@ -752,7 +770,7 @@ impl RangeIndexer {
|
|||||||
for (upload, list) in uploads_of(by_release.remove(&key).unwrap()).into_iter().enumerate() {
|
for (upload, list) in uploads_of(by_release.remove(&key).unwrap()).into_iter().enumerate() {
|
||||||
let judged: Vec<JudgedFile> = list
|
let judged: Vec<JudgedFile> = list
|
||||||
.iter()
|
.iter()
|
||||||
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id })
|
.map(|f| JudgedFile { name: &f[0].file.name, total_parts: f[0].file.total_parts, groups: &f[0].file.groups, first_message_id: &f[0].message_id, poster: &f[0].file.poster })
|
||||||
.collect();
|
.collect();
|
||||||
match names::judge(&key, &judged, &self.rules) {
|
match names::judge(&key, &judged, &self.rules) {
|
||||||
Verdict::Keep => {
|
Verdict::Keep => {
|
||||||
@@ -1015,3 +1033,18 @@ mod beside_quotes_tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod posted_span_tests {
|
||||||
|
use super::posted_span;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn without_the_earliest_and_latest_percent() {
|
||||||
|
assert_eq!(posted_span(vec![]), (None, None));
|
||||||
|
let mut dates: Vec<i64> = (0..200).map(|i| 1_000_000 + i).collect();
|
||||||
|
dates.push(5);
|
||||||
|
dates.push(9_999_999_999);
|
||||||
|
// 202 dates: 2 trimmed at each end, so the two wild ones go (postedSpan).
|
||||||
|
assert_eq!(posted_span(dates), (Some(1_000_001), Some(1_000_198)));
|
||||||
|
}
|
||||||
|
}
|
||||||
+5
-1
@@ -96,7 +96,11 @@ fn dump_batch(batch: &Batch, out: &mut impl Write) {
|
|||||||
out.write_all(line.as_bytes()).unwrap();
|
out.write_all(line.as_bytes()).unwrap();
|
||||||
line.clear();
|
line.clear();
|
||||||
};
|
};
|
||||||
line.push_str(&format!("{{\"t\":\"b\",\"from\":{},\"next\":{},\"headers\":{}}}", batch.from, batch.next, batch.headers));
|
let opt = |t: Option<i64>| t.map_or("null".to_string(), |t| t.to_string());
|
||||||
|
line.push_str(&format!(
|
||||||
|
"{{\"t\":\"b\",\"from\":{},\"next\":{},\"headers\":{},\"postedLow\":{},\"postedHigh\":{}}}",
|
||||||
|
batch.from, batch.next, batch.headers, opt(batch.posted_low), opt(batch.posted_high)
|
||||||
|
));
|
||||||
emit(&mut line);
|
emit(&mut line);
|
||||||
let release = |r: &indexer::ReleaseChunk, kind: &str, line: &mut String| {
|
let release = |r: &indexer::ReleaseChunk, kind: &str, line: &mut String| {
|
||||||
line.push_str(&format!("{{\"t\":\"{kind}\",\"upload\":{},\"key\":", r.upload));
|
line.push_str(&format!("{{\"t\":\"{kind}\",\"upload\":{},\"key\":", r.upload));
|
||||||
|
|||||||
+78
-2
@@ -744,8 +744,25 @@ pub fn size_after_name(subject: &str, name: &str) -> String {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A name posted unquoted with its size in bytes right after it, at the subject's end ("… Nude Celebrity Pictures <<<<< -
|
||||||
|
/// !index0001.jpg 75220 bytes"): a tool that posts a folder under the subject typed for it (sizedName). Not a quoted name.
|
||||||
|
pub fn sized_name(subject: &str, name: &str) -> bool {
|
||||||
|
let s = subject.trim_end();
|
||||||
|
let lower = s.to_ascii_lowercase();
|
||||||
|
let unit = if lower.ends_with("bytes") { 5 } else if lower.ends_with("byte") { 4 } else { return false };
|
||||||
|
let s = s[..s.len() - unit].trim_end();
|
||||||
|
// /\d[\d.,]*$/: from the first digit of the trailing run of digits, dots and commas.
|
||||||
|
let run = s.len() - s.trim_end_matches(|c: char| c.is_ascii_digit() || c == '.' || c == ',').len();
|
||||||
|
let tail = &s[s.len() - run..];
|
||||||
|
let Some(first) = tail.find(|c: char| c.is_ascii_digit()) else { return false };
|
||||||
|
let s = s[..s.len() - run + first].trim_end();
|
||||||
|
let s = s.strip_suffix('-').unwrap_or(s).trim_end();
|
||||||
|
!name.is_empty() && s.ends_with(name) && !s.ends_with(&format!("\"{name}\""))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
||||||
let poster = if random_poster(poster) { "" } else { poster };
|
let poster = if random_poster(poster) { "" } else { poster };
|
||||||
|
let sized = sized_name(subject, name);
|
||||||
// The post rule's key takes the [i/M] total; the release's file total any counter's (counterTotal).
|
// The post rule's key takes the [i/M] total; the release's file total any counter's (counterTotal).
|
||||||
let post_total = file_counter_total(subject);
|
let post_total = file_counter_total(subject);
|
||||||
let stem = random_volume_base(file_stem(run_on(name)));
|
let stem = random_volume_base(file_stem(run_on(name)));
|
||||||
@@ -813,7 +830,8 @@ pub fn release_identity(poster: &str, subject: &str, name: &str) -> Release {
|
|||||||
let one_word = image && remainder.len() >= 3 && remainder.bytes().all(|c| c.is_ascii_alphabetic());
|
let one_word = image && remainder.len() >= 3 && remainder.bytes().all(|c| c.is_ascii_alphabetic());
|
||||||
if !tag && (names_release(&remainder) || one_word) {
|
if !tag && (names_release(&remainder) || one_word) {
|
||||||
// Without a counter an image's set: its name up to its last number, digits kept; none without numbers.
|
// Without a counter an image's set: its name up to its last number, digits kept; none without numbers.
|
||||||
let bound = total.clone().unwrap_or_else(|| if image { image_set(&stem) } else { String::new() });
|
// A folder's images posted with their sizes: the subject names the folder, the images no set.
|
||||||
|
let bound = total.clone().unwrap_or_else(|| if image && !sized { image_set(&stem) } else { String::new() });
|
||||||
let mut parts: Vec<&str> = vec!["subject", poster, &remainder, &bound];
|
let mut parts: Vec<&str> = vec!["subject", poster, &remainder, &bound];
|
||||||
// An upload's random stem on its volumes ("Z3mMTMPsoT4ikE_Di3fKnNF21.part01.rar" under "[ TOWN ] … [ TV ]").
|
// An upload's random stem on its volumes ("Z3mMTMPsoT4ikE_Di3fKnNF21.part01.rar" under "[ TOWN ] … [ TV ]").
|
||||||
let stripped: String = stem.chars().filter(|c| !matches!(c, '-' | '_')).collect();
|
let stripped: String = stem.chars().filter(|c| !matches!(c, '-' | '_')).collect();
|
||||||
@@ -1223,6 +1241,7 @@ pub struct JudgedFile<'a> {
|
|||||||
pub total_parts: u32,
|
pub total_parts: u32,
|
||||||
pub groups: &'a [String],
|
pub groups: &'a [String],
|
||||||
pub first_message_id: &'a str,
|
pub first_message_id: &'a str,
|
||||||
|
pub poster: &'a str,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub enum Verdict {
|
pub enum Verdict {
|
||||||
@@ -1242,6 +1261,8 @@ pub struct Rules {
|
|||||||
pub per_article_max_parts: u32,
|
pub per_article_max_parts: u32,
|
||||||
pub sample_percent: f64,
|
pub sample_percent: f64,
|
||||||
pub drop_executables: bool,
|
pub drop_executables: bool,
|
||||||
|
/// Fragments of the poster lowercased without punctuation (droppedPosters).
|
||||||
|
pub dropped_posters: Vec<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Rules {
|
impl Rules {
|
||||||
@@ -1255,13 +1276,25 @@ impl Rules {
|
|||||||
per_article_max_parts: 3,
|
per_article_max_parts: 3,
|
||||||
sample_percent: 1.0,
|
sample_percent: 1.0,
|
||||||
drop_executables: true,
|
drop_executables: true,
|
||||||
|
dropped_posters: ["celebmovie", "celebwow", "celebfx", "iluvcelebs", "nudecelebs", "celebsextapes"].iter().map(|p| p.to_string()).collect(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// posterWords: lowercased, letters and digits only.
|
||||||
|
pub fn poster_words(poster: &str) -> String {
|
||||||
|
poster.chars().filter(|c| c.is_ascii_alphanumeric()).map(|c| c.to_ascii_lowercase()).collect()
|
||||||
|
}
|
||||||
|
|
||||||
/// RuleSet.judge.
|
/// RuleSet.judge.
|
||||||
pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict {
|
pub fn judge(release_key: &str, files: &[JudgedFile], rules: &Rules) -> Verdict {
|
||||||
let sample = explored(release_key, rules.sample_percent);
|
let sample = explored(release_key, rules.sample_percent);
|
||||||
|
if let (false, Some(first)) = (rules.dropped_posters.is_empty(), files.first()) {
|
||||||
|
let words = poster_words(first.poster);
|
||||||
|
if let Some(dropped) = rules.dropped_posters.iter().find(|p| words.contains(p.as_str())) {
|
||||||
|
return Verdict::Drop { rule: "poster", key: dropped.clone(), sample };
|
||||||
|
}
|
||||||
|
}
|
||||||
let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect();
|
let content: Vec<&JudgedFile> = files.iter().filter(|f| !test(&SIDECAR, f.name)).collect();
|
||||||
if rules.drop_executables && !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) {
|
if rules.drop_executables && !content.is_empty() && content.iter().all(|f| test(&EXECUTABLE, f.name)) {
|
||||||
let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap();
|
let c = EXECUTABLE.captures(content[0].name.as_bytes()).unwrap();
|
||||||
@@ -1441,9 +1474,36 @@ mod numbered_tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod poster_rule_tests {
|
||||||
|
use super::{judge, JudgedFile, Rules, Verdict};
|
||||||
|
|
||||||
|
fn verdict(poster: &str, rules: &Rules) -> Option<(&'static str, String)> {
|
||||||
|
let groups = vec!["alt.binaries.erotica".to_string()];
|
||||||
|
let files = [JudgedFile { name: "!index0001.jpg", total_parts: 1, groups: &groups, first_message_id: "<a@b>", poster }];
|
||||||
|
match judge("k", &files, rules) {
|
||||||
|
Verdict::Keep => None,
|
||||||
|
Verdict::Drop { rule, key, .. } => Some((rule, key)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_dropped_posters_uploads_are_dropped() {
|
||||||
|
let rules = Rules::defaults(Default::default());
|
||||||
|
assert_eq!(verdict("@_CelebMovie_@", &rules), Some(("poster", "celebmovie".to_string())));
|
||||||
|
assert_eq!(verdict("@__Celeb_Movie__@", &rules), Some(("poster", "celebmovie".to_string())));
|
||||||
|
assert_eq!(verdict("NudeCelebs@CelebWow.com", &rules), Some(("poster", "celebwow".to_string())));
|
||||||
|
assert_eq!(verdict("@nude-celebs", &rules), Some(("poster", "nudecelebs".to_string())));
|
||||||
|
assert_eq!(verdict("celeborn@not.here (Celeborn)", &rules), None);
|
||||||
|
assert_eq!(verdict(r#""CelebTA.com" <from@celebta.com>"#, &rules), None);
|
||||||
|
let none = Rules { dropped_posters: vec![], ..Rules::defaults(Default::default()) };
|
||||||
|
assert_eq!(verdict("@_CelebMovie_@", &none), None);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod image_set_tests {
|
mod image_set_tests {
|
||||||
use super::{release_identity, subject_remainder, subject_remainder_re};
|
use super::{release_identity, sized_name, subject_remainder, subject_remainder_re};
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn random_posters_are_no_part_of_the_key() {
|
fn random_posters_are_no_part_of_the_key() {
|
||||||
@@ -1502,6 +1562,22 @@ mod image_set_tests {
|
|||||||
assert_eq!(mint("JimAndJoey Kindly Husband-to-Be.JPG").key, mint("Oops OnlyOneMore.JPG").key);
|
assert_eq!(mint("JimAndJoey Kindly Husband-to-Be.JPG").key, mint("Oops OnlyOneMore.JPG").key);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn a_folders_images_posted_with_their_sizes_are_one_release() {
|
||||||
|
let celeb = |n: &str, size: u32| release_identity("@_CelebMovie_@", &format!(">>>>> Jamie Lee Curtis - Nude Celebrity Pictures <<<<< - {n} {size} bytes"), n);
|
||||||
|
let keys: std::collections::HashSet<String> =
|
||||||
|
["!index0001.jpg", "00-00-02_123_842lo.jpg", "00_00_13.jpg", "bboobs02-curtis-jl.jpg", "0c4_jamie_lee_curtis_15.jpg"].iter().enumerate().map(|(i, n)| celeb(n, 75220 + i as u32).key).collect();
|
||||||
|
assert_eq!(keys.len(), 1);
|
||||||
|
assert_eq!(celeb("00_00_13.jpg", 58748).source, "subject");
|
||||||
|
assert_ne!(celeb("00_00_13.jpg", 1).key, release_identity("@_CelebMovie_@", ">>>>> Carmen Electra - Xmas Nudes <<<<< - 00_00_13.jpg 1 bytes", "00_00_13.jpg").key);
|
||||||
|
let ctp = |n: &str| release_identity("new@update.org (New CTP update)", &format!("*** NEW *** CTP update & adds - {n} - 18441 bytes"), n);
|
||||||
|
assert_eq!(ctp("cc_CG2008_v10_01_Stephanie_Emma.jpg").key, ctp("cc_CastingCall_LasVegas_01_Kristin_00_Header1.jpg").key);
|
||||||
|
assert!(sized_name("x - a.jpg 1,234 Bytes", "a.jpg"));
|
||||||
|
assert!(!sized_name(r#"x - "a.jpg" yEnc 320794 bytes"#, "a.jpg"));
|
||||||
|
assert!(!sized_name("x - a.jpg 5 megabytes", "a.jpg"));
|
||||||
|
assert!(!sized_name("x - a.jpg yEnc", "a.jpg"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn other_quoted_text_and_text_separators_name_the_release() {
|
fn other_quoted_text_and_text_separators_name_the_release() {
|
||||||
let cc = |name: &str, n: &str| release_identity("p", &format!(r#">>> CREATIVE COMMONS NZB <<< "{name} " - File 1 of 9: "creative_commons_nzb_{n}.rar" yEnc"#), &format!("creative_commons_nzb_{n}.rar"));
|
let cc = |name: &str, n: &str| release_identity("p", &format!(r#">>> CREATIVE COMMONS NZB <<< "{name} " - File 1 of 9: "creative_commons_nzb_{n}.rar" yEnc"#), &format!("creative_commons_nzb_{n}.rar"));
|
||||||
|
|||||||
@@ -74,10 +74,13 @@ struct WireFragment<'a> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
struct WireBatch<'a> {
|
struct WireBatch<'a> {
|
||||||
from: i64,
|
from: i64,
|
||||||
next: i64,
|
next: i64,
|
||||||
headers: u64,
|
headers: u64,
|
||||||
|
posted_low: Option<i64>,
|
||||||
|
posted_high: Option<i64>,
|
||||||
releases: Vec<WireRelease<'a>>,
|
releases: Vec<WireRelease<'a>>,
|
||||||
samples: Vec<WireRelease<'a>>,
|
samples: Vec<WireRelease<'a>>,
|
||||||
candidates: Vec<WireCandidate<'a>>,
|
candidates: Vec<WireCandidate<'a>>,
|
||||||
@@ -134,6 +137,8 @@ pub fn to_json(batch: &Batch) -> Vec<u8> {
|
|||||||
from: batch.from,
|
from: batch.from,
|
||||||
next: batch.next,
|
next: batch.next,
|
||||||
headers: batch.headers,
|
headers: batch.headers,
|
||||||
|
posted_low: batch.posted_low,
|
||||||
|
posted_high: batch.posted_high,
|
||||||
releases: batch.releases.iter().map(|r| release(r, None)).collect(),
|
releases: batch.releases.iter().map(|r| release(r, None)).collect(),
|
||||||
samples: batch.samples.iter().map(|(r, rule, key)| release(r, Some((rule, key.as_str())))).collect(),
|
samples: batch.samples.iter().map(|(r, rule, key)| release(r, Some((rule, key.as_str())))).collect(),
|
||||||
candidates: batch.candidates.iter().map(candidate).collect(),
|
candidates: batch.candidates.iter().map(candidate).collect(),
|
||||||
|
|||||||
+1
-1
@@ -1,4 +1,4 @@
|
|||||||
export { RangeIndexer, chunkOf, candidatesOf, releaseKeyOf, uploadsOf, UPLOAD_GAP_MS, type EdgeBatch, type ReleaseChunk, type DroppedChunk, type ProbeCandidate, type CandidateKind, type RuleCount, type Fragment, type RangeIndexerOptions } from './range_indexer.js'
|
export { RangeIndexer, postedSpan, chunkOf, candidatesOf, releaseKeyOf, uploadsOf, UPLOAD_GAP_MS, type EdgeBatch, type ReleaseChunk, type DroppedChunk, type ProbeCandidate, type CandidateKind, type RuleCount, type Fragment, type RangeIndexerOptions } from './range_indexer.js'
|
||||||
export { encodeFrame, decodeFrames, isKeepalive, KEEPALIVE, type Frame } from './stream.js'
|
export { encodeFrame, decodeFrames, isKeepalive, KEEPALIVE, type Frame } from './stream.js'
|
||||||
export { toWire, fromWire, type WireBatch } from './wire.js'
|
export { toWire, fromWire, type WireBatch } from './wire.js'
|
||||||
export type { Lease } from './lease.js'
|
export type { Lease } from './lease.js'
|
||||||
|
|||||||
@@ -48,7 +48,7 @@ export interface ProbeCandidate {
|
|||||||
|
|
||||||
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
|
/** What a rule dropped in a batch (REBUILD.md, rules and recovery). */
|
||||||
export interface RuleCount {
|
export interface RuleCount {
|
||||||
rule: 'crosspost' | 'obfuscated' | 'executable' | 'garbage' | 'per-article' | 'incomplete'
|
rule: 'crosspost' | 'obfuscated' | 'poster' | 'executable' | 'garbage' | 'per-article' | 'incomplete'
|
||||||
key: string
|
key: string
|
||||||
articles: number
|
articles: number
|
||||||
files: number
|
files: number
|
||||||
@@ -77,6 +77,13 @@ export interface EdgeBatch {
|
|||||||
/** First article number not covered yet. */
|
/** First article number not covered yet. */
|
||||||
next: number
|
next: number
|
||||||
headers: number
|
headers: number
|
||||||
|
/**
|
||||||
|
* The batch's posting times without its earliest and latest 1% (ms; null without a dated article): the range's
|
||||||
|
* time span is their min and max (headers dated years off don't stretch it). The settler's final completeness
|
||||||
|
* reads it (LOG.md 2026-10-16 (67)).
|
||||||
|
*/
|
||||||
|
postedLow: number | null
|
||||||
|
postedHigh: number | null
|
||||||
releases: ReleaseChunk[]
|
releases: ReleaseChunk[]
|
||||||
samples: DroppedChunk[]
|
samples: DroppedChunk[]
|
||||||
candidates: ProbeCandidate[]
|
candidates: ProbeCandidate[]
|
||||||
@@ -115,6 +122,8 @@ export class RangeIndexer {
|
|||||||
private aggregator = new FileAggregator<EdgeSegment>(UPLOAD_GAP_MS)
|
private aggregator = new FileAggregator<EdgeSegment>(UPLOAD_GAP_MS)
|
||||||
private scanner = new SegmentScanner()
|
private scanner = new SegmentScanner()
|
||||||
private segments: EdgeSegment[] = []
|
private segments: EdgeSegment[] = []
|
||||||
|
/** Posting times of this batch's segments (postedSpan at take). */
|
||||||
|
private dates: number[] = []
|
||||||
private rules: RuleSet
|
private rules: RuleSet
|
||||||
private batch: EdgeBatch
|
private batch: EdgeBatch
|
||||||
private counts = new Map<string, RuleCount>()
|
private counts = new Map<string, RuleCount>()
|
||||||
@@ -155,8 +164,10 @@ export class RangeIndexer {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
const segment = this.scanner.segment(articleNumber, subject, from, date, messageId, bytes, xref, group)
|
const segment = this.scanner.segment(articleNumber, subject, from, date, messageId, bytes, xref, group)
|
||||||
if (segment) this.segments.push(segment)
|
if (segment) {
|
||||||
else this.count('obfuscated', '', 1, 0)
|
this.segments.push(segment)
|
||||||
|
if (Number.isFinite(segment.postedAt)) this.dates.push(segment.postedAt)
|
||||||
|
} else this.count('obfuscated', '', 1, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
private endChunk(next: number) {
|
private endChunk(next: number) {
|
||||||
@@ -180,6 +191,8 @@ export class RangeIndexer {
|
|||||||
const out = this.batch
|
const out = this.batch
|
||||||
out.counts = [...this.counts.values()]
|
out.counts = [...this.counts.values()]
|
||||||
this.counts.clear()
|
this.counts.clear()
|
||||||
|
;[out.postedLow, out.postedHigh] = postedSpan(this.dates)
|
||||||
|
this.dates = []
|
||||||
this.batch = this.empty(out.next)
|
this.batch = this.empty(out.next)
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
@@ -215,7 +228,7 @@ export class RangeIndexer {
|
|||||||
uploadsOf(all).forEach((list, upload) => {
|
uploadsOf(all).forEach((list, upload) => {
|
||||||
const verdict = this.rules.judge(
|
const verdict = this.rules.judge(
|
||||||
key,
|
key,
|
||||||
list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId }))
|
list.map((f) => ({ name: f.segments[0].file.name, totalParts: f.segments[0].totalParts, groups: f.segments[0].file.groups, firstMessageId: f.segments[0].messageId, poster: f.segments[0].file.poster }))
|
||||||
)
|
)
|
||||||
if (verdict.rule === null) {
|
if (verdict.rule === null) {
|
||||||
this.batch.releases.push(chunkOf(key, list, upload))
|
this.batch.releases.push(chunkOf(key, list, upload))
|
||||||
@@ -290,10 +303,18 @@ export class RangeIndexer {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private empty(from: number): EdgeBatch {
|
private empty(from: number): EdgeBatch {
|
||||||
return { from, next: from, headers: 0, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
|
return { from, next: from, headers: 0, postedLow: null, postedHigh: null, releases: [], samples: [], candidates: [], counts: [], fragments: [] }
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Posting times without the earliest and latest 1% (floor(n / 100) at each end); nulls without any. */
|
||||||
|
export function postedSpan(dates: number[]): [number | null, number | null] {
|
||||||
|
if (dates.length === 0) return [null, null]
|
||||||
|
const sorted = Float64Array.from(dates).sort()
|
||||||
|
const k = Math.floor(sorted.length / 100)
|
||||||
|
return [sorted[k], sorted[sorted.length - 1 - k]]
|
||||||
|
}
|
||||||
|
|
||||||
/** The release key; a file without a recognisable release becomes a release of its own. */
|
/** The release key; a file without a recognisable release becomes a release of its own. */
|
||||||
export function releaseKeyOf(s: EdgeSegment): string {
|
export function releaseKeyOf(s: EdgeSegment): string {
|
||||||
return s.file.release?.key ?? hash('sha1', `file\n${s.fileKey}`)
|
return s.file.release?.key ?? hash('sha1', `file\n${s.fileKey}`)
|
||||||
|
|||||||
@@ -2,4 +2,4 @@
|
|||||||
* Newznab API formatting, independent of AdonisJS. From voyager 1 (its CONCEPT.md D23).
|
* Newznab API formatting, independent of AdonisJS. From voyager 1 (its CONCEPT.md D23).
|
||||||
*/
|
*/
|
||||||
export { CATEGORIES, categoryLabel, expandCategories, parentCategory } from './categories.js'
|
export { CATEGORIES, categoryLabel, expandCategories, parentCategory } from './categories.js'
|
||||||
export { capsXml, errorXml, resultsXml, type NewznabItem, type NewznabServer } from './xml.js'
|
export { capsXml, errorXml, resultsXml, type ApiLimits, type NewznabItem, type NewznabServer } from './xml.js'
|
||||||
+24
-4
@@ -27,7 +27,7 @@ export interface NewznabServer {
|
|||||||
apiKey?: string
|
apiKey?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Newznab error codes: 100 credentials, 200/201 parameters, 202 function, 300 item. */
|
/** Newznab error codes: 100 credentials, 101 suspended, 102 privileges, 200/201 parameters, 202 function, 300 item, 500/501 request/download limit. */
|
||||||
/** Spotnet UBB markup (or the HTML of old spots) as plain text: [br] as line breaks, other tags dropped. */
|
/** Spotnet UBB markup (or the HTML of old spots) as plain text: [br] as line breaks, other tags dropped. */
|
||||||
export function ubbToText(ubb: string): string {
|
export function ubbToText(ubb: string): string {
|
||||||
return ubb
|
return ubb
|
||||||
@@ -44,11 +44,23 @@ export function errorXml(code: number, description: string): string {
|
|||||||
.end({ prettyPrint: true })
|
.end({ prettyPrint: true })
|
||||||
}
|
}
|
||||||
|
|
||||||
export function capsXml(server: NewznabServer, limits: { max: number; default: number }): string {
|
/** A caller's day so far and its limits (null: none), as nZEDb-style indexers report them. */
|
||||||
|
export interface ApiLimits {
|
||||||
|
apiCurrent: number
|
||||||
|
apiMax: number | null
|
||||||
|
grabCurrent: number
|
||||||
|
grabMax: number | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export function capsXml(
|
||||||
|
server: NewznabServer,
|
||||||
|
limits: { max: number; default: number },
|
||||||
|
registration = { available: false, open: false }
|
||||||
|
): string {
|
||||||
const caps = create({ version: '1.0', encoding: 'UTF-8' }).ele('caps')
|
const caps = create({ version: '1.0', encoding: 'UTF-8' }).ele('caps')
|
||||||
caps.ele('server', { version: '1.0', title: server.title, strapline: 'Usenet indexer', url: `${server.baseUrl}/` })
|
caps.ele('server', { version: '1.0', title: server.title, strapline: 'Usenet indexer', url: `${server.baseUrl}/` })
|
||||||
caps.ele('limits', { max: String(limits.max), default: String(limits.default) })
|
caps.ele('limits', { max: String(limits.max), default: String(limits.default) })
|
||||||
caps.ele('registration', { available: 'no', open: 'no' })
|
caps.ele('registration', { available: registration.available ? 'yes' : 'no', open: registration.open ? 'yes' : 'no' })
|
||||||
const searching = caps.ele('searching')
|
const searching = caps.ele('searching')
|
||||||
searching.ele('search', { available: 'yes', supportedParams: 'q' })
|
searching.ele('search', { available: 'yes', supportedParams: 'q' })
|
||||||
searching.ele('tv-search', { available: 'yes', supportedParams: 'q,season,ep' })
|
searching.ele('tv-search', { available: 'yes', supportedParams: 'q,season,ep' })
|
||||||
@@ -63,7 +75,7 @@ export function capsXml(server: NewznabServer, limits: { max: number; default: n
|
|||||||
return caps.end({ prettyPrint: true })
|
return caps.end({ prettyPrint: true })
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resultsXml(server: NewznabServer, items: NewznabItem[], offset: number, total: number): string {
|
export function resultsXml(server: NewznabServer, items: NewznabItem[], offset: number, total: number, limits?: ApiLimits): string {
|
||||||
const rss = create({ version: '1.0', encoding: 'UTF-8' }).ele('rss', {
|
const rss = create({ version: '1.0', encoding: 'UTF-8' }).ele('rss', {
|
||||||
'version': '2.0',
|
'version': '2.0',
|
||||||
'xmlns:atom': 'http://www.w3.org/2005/Atom',
|
'xmlns:atom': 'http://www.w3.org/2005/Atom',
|
||||||
@@ -74,6 +86,14 @@ export function resultsXml(server: NewznabServer, items: NewznabItem[], offset:
|
|||||||
channel.ele('description').txt(`${server.title} API results`)
|
channel.ele('description').txt(`${server.title} API results`)
|
||||||
channel.ele('link').txt(`${server.baseUrl}/`)
|
channel.ele('link').txt(`${server.baseUrl}/`)
|
||||||
channel.ele('newznab:response', { offset: String(offset), total: String(total) })
|
channel.ele('newznab:response', { offset: String(offset), total: String(total) })
|
||||||
|
if (limits) {
|
||||||
|
channel.ele('newznab:apilimits', {
|
||||||
|
apicurrent: String(limits.apiCurrent),
|
||||||
|
...(limits.apiMax !== null && { apimax: String(limits.apiMax) }),
|
||||||
|
grabcurrent: String(limits.grabCurrent),
|
||||||
|
...(limits.grabMax !== null && { grabmax: String(limits.grabMax) }),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
const key = server.apiKey ? `&apikey=${encodeURIComponent(server.apiKey)}` : ''
|
const key = server.apiKey ? `&apikey=${encodeURIComponent(server.apiKey)}` : ''
|
||||||
for (const item of items) {
|
for (const item of items) {
|
||||||
|
|||||||
+21
-1
@@ -332,6 +332,23 @@ export function runOn(name: string): string {
|
|||||||
return name
|
return name
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A name posted unquoted with its size in bytes right after it, at the subject's end ("… Nude Celebrity Pictures <<<<< -
|
||||||
|
* !index0001.jpg 75220 bytes", "*** NEW *** CTP update & adds - cc_CG2008_v10_01_Stephanie_Emma.jpg - 18441 bytes"):
|
||||||
|
* a tool that posts a folder under the subject typed for it, file by file. Not a quoted name ("The excellent STIRILING
|
||||||
|
* SILVER - Always popular - "stp250x001.jpg" yEnc 320794 bytes").
|
||||||
|
*/
|
||||||
|
export function sizedName(subject: string, name: string): boolean {
|
||||||
|
let s = subject.trimEnd()
|
||||||
|
const unit = /bytes?$/i.exec(s)
|
||||||
|
if (!unit) return false
|
||||||
|
s = s.slice(0, unit.index).trimEnd()
|
||||||
|
const digits = /\d[\d.,]*$/.exec(s)
|
||||||
|
if (!digits) return false
|
||||||
|
s = s.slice(0, digits.index).trimEnd().replace(/-$/, '').trimEnd()
|
||||||
|
return name.length > 0 && s.endsWith(name) && !s.endsWith(`"${name}"`)
|
||||||
|
}
|
||||||
|
|
||||||
/** The subject without a number after the file's unquoted name at its end (" - 28": a size in kB). */
|
/** The subject without a number after the file's unquoted name at its end (" - 28": a size in kB). */
|
||||||
export function sizeAfterName(subject: string, name: string): string {
|
export function sizeAfterName(subject: string, name: string): string {
|
||||||
const end = subject.trimEnd()
|
const end = subject.trimEnd()
|
||||||
@@ -419,7 +436,10 @@ export function releaseIdentity(input: ReleaseInput): ReleaseIdentity | null {
|
|||||||
// Without a counter an image's set bounds it: its name up to its last number, digits kept ("stp250x001.jpg"
|
// Without a counter an image's set bounds it: its name up to its last number, digits kept ("stp250x001.jpg"
|
||||||
// → "stp250x": STIRILING SILVER's 300 sets were one release of 44k images); a name without numbers names no
|
// → "stp250x": STIRILING SILVER's 300 sets were one release of 44k images); a name without numbers names no
|
||||||
// set ("Kindly Husband-to-Be.JPG": a thread of 1,177 releases), so the subject alone (LOG.md 2026-10-12 (2)).
|
// set ("Kindly Husband-to-Be.JPG": a thread of 1,177 releases), so the subject alone (LOG.md 2026-10-12 (2)).
|
||||||
const set = image ? imageSet(stem) : ''
|
// A folder's images posted with their sizes: the subject names the folder, the images no set (a celebrity's
|
||||||
|
// scraped pictures, "!index0001.jpg", "00_00_13.jpg", "bboobs02-curtis-jl.jpg", were 219 releases of 1,423 images;
|
||||||
|
// LOG.md 2026-10-16 (63)).
|
||||||
|
const set = image && !sizedName(input.subject, input.name) ? imageSet(stem) : ''
|
||||||
const parts: (string | number)[] = ['subject', poster, remainder, total ?? set]
|
const parts: (string | number)[] = ['subject', poster, remainder, total ?? set]
|
||||||
// An upload's random stem on its volumes ("[ TOWN ] [ www town ag ] [ TV ] … "Z3mMTMPsoT4ikE_Di3fKnNF21.part01.rar"",
|
// An upload's random stem on its volumes ("[ TOWN ] [ www town ag ] [ TV ] … "Z3mMTMPsoT4ikE_Di3fKnNF21.part01.rar"",
|
||||||
// "….vol4+4.par2"): under a constant tag every upload of a total was one release (2,338 files for 20; LOG.md
|
// "….vol4+4.par2"): under a constant tag every upload of a total was one release (2,338 files for 20; LOG.md
|
||||||
|
|||||||
+1
-1
@@ -1,2 +1,2 @@
|
|||||||
export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
|
export { uploadTool, signatureGroup, signatureKey, explored } from './upload_tools.js'
|
||||||
export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, type Rules, type Verdict } from './rule_set.js'
|
export { RuleSet, DEFAULT_RULES, isRandomFile, ownerOf, posterWords, type Rules, type Verdict } from './rule_set.js'
|
||||||
+22
-3
@@ -24,6 +24,12 @@ export interface Rules {
|
|||||||
* 2026-10-09). Genuine software comes in archives.
|
* 2026-10-09). Genuine software comes in archives.
|
||||||
*/
|
*/
|
||||||
dropExecutables: boolean
|
dropExecutables: boolean
|
||||||
|
/**
|
||||||
|
* Posters whose uploads are dropped whatever they are: fragments of the poster lowercased without punctuation
|
||||||
|
* ("@__Celeb_Movie__@" → "celebmovie"). The CelebMovie family's scraped celebrity pictures, the same files again and
|
||||||
|
* again under other titles (119k releases on 2026-10-16; LOG.md 2026-10-16 (65)).
|
||||||
|
*/
|
||||||
|
droppedPosters: string[]
|
||||||
}
|
}
|
||||||
|
|
||||||
export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
||||||
@@ -32,9 +38,15 @@ export const DEFAULT_RULES: Omit<Rules, 'priorities' | 'droppedSignatures'> = {
|
|||||||
perArticleMaxParts: 3,
|
perArticleMaxParts: 3,
|
||||||
samplePercent: 1,
|
samplePercent: 1,
|
||||||
dropExecutables: true,
|
dropExecutables: true,
|
||||||
|
droppedPosters: ['celebmovie', 'celebwow', 'celebfx', 'iluvcelebs', 'nudecelebs', 'celebsextapes'],
|
||||||
}
|
}
|
||||||
|
|
||||||
export type Verdict = { rule: null } | { rule: 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean }
|
export type Verdict = { rule: null } | { rule: 'poster' | 'executable' | 'garbage' | 'per-article'; key: string; sample: boolean }
|
||||||
|
|
||||||
|
/** A poster as droppedPosters matches it: lowercased, letters and digits only. */
|
||||||
|
export function posterWords(poster: string): string {
|
||||||
|
return poster.replace(/[^A-Za-z0-9]/g, '').toLowerCase()
|
||||||
|
}
|
||||||
|
|
||||||
/** Files that come along with a release's content: they don't make it one of executables or not. */
|
/** Files that come along with a release's content: they don't make it one of executables or not. */
|
||||||
const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i
|
const SIDECAR = /\.(par2|nfo|sfv|txt|jpe?g|png|nzb)$/i
|
||||||
@@ -57,18 +69,25 @@ export function ownerOf(groups: string[], priorities: Record<string, number>): s
|
|||||||
export class RuleSet {
|
export class RuleSet {
|
||||||
private signatures: Set<string>
|
private signatures: Set<string>
|
||||||
private tools: Set<string>
|
private tools: Set<string>
|
||||||
|
private posters: string[]
|
||||||
constructor(readonly rules: Rules) {
|
constructor(readonly rules: Rules) {
|
||||||
|
this.posters = rules.droppedPosters ?? []
|
||||||
this.signatures = new Set(rules.droppedSignatures)
|
this.signatures = new Set(rules.droppedSignatures)
|
||||||
this.tools = new Set(rules.droppedTools)
|
this.tools = new Set(rules.droppedTools)
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A release's verdict from its files: dropped when its content is bare
|
* A release's verdict from its files: dropped when its poster is; when its content is bare
|
||||||
* executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is
|
* executables; else kept as a whole when any file has a readable name; else dropped as garbage when its signature or tool is
|
||||||
* dropped, as per-article random subjects when every file has few parts.
|
* dropped, as per-article random subjects when every file has few parts.
|
||||||
*/
|
*/
|
||||||
judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string }[]): Verdict {
|
judge(releaseKey: string, files: { name: string; totalParts: number; groups: string[]; firstMessageId: string; poster: string }[]): Verdict {
|
||||||
const sample = explored(releaseKey, this.rules.samplePercent)
|
const sample = explored(releaseKey, this.rules.samplePercent)
|
||||||
|
if (this.posters.length > 0 && files.length > 0) {
|
||||||
|
const words = posterWords(files[0].poster)
|
||||||
|
const dropped = this.posters.find((p) => words.includes(p))
|
||||||
|
if (dropped) return { rule: 'poster', key: dropped, sample }
|
||||||
|
}
|
||||||
if (this.rules.dropExecutables) {
|
if (this.rules.dropExecutables) {
|
||||||
const content = files.filter((f) => !SIDECAR.test(f.name))
|
const content = files.filter((f) => !SIDECAR.test(f.name))
|
||||||
if (content.length > 0 && content.every((f) => EXECUTABLE.test(f.name))) {
|
if (content.length > 0 && content.every((f) => EXECUTABLE.test(f.name))) {
|
||||||
|
|||||||
+126
@@ -891,3 +891,129 @@ kbd {
|
|||||||
margin-top: 4px;
|
margin-top: 4px;
|
||||||
color: var(--faint);
|
color: var(--faint);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/* --- Account (ACCESS=accounts) ------------------------------------------ */
|
||||||
|
|
||||||
|
.account-button {
|
||||||
|
height: 32px;
|
||||||
|
padding: 0 12px;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.account-button .tier {
|
||||||
|
color: var(--accent);
|
||||||
|
text-transform: uppercase;
|
||||||
|
font-size: 11px;
|
||||||
|
letter-spacing: 0.04em;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dialog {
|
||||||
|
width: min(420px, calc(100vw - 32px));
|
||||||
|
padding: 0;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: var(--radius);
|
||||||
|
background: var(--surface);
|
||||||
|
color: var(--text);
|
||||||
|
box-shadow: var(--shadow);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dialog::backdrop {
|
||||||
|
background: rgba(10, 14, 20, 0.35);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dialog-head {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: space-between;
|
||||||
|
padding: 14px 18px;
|
||||||
|
border-bottom: 1px solid var(--border);
|
||||||
|
}
|
||||||
|
|
||||||
|
.dialog-head h2 {
|
||||||
|
margin: 0;
|
||||||
|
font-size: 16px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.dialog-body {
|
||||||
|
padding: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.form {
|
||||||
|
display: grid;
|
||||||
|
gap: 12px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.form label {
|
||||||
|
display: grid;
|
||||||
|
gap: 4px;
|
||||||
|
font-size: 13px;
|
||||||
|
color: var(--muted);
|
||||||
|
}
|
||||||
|
|
||||||
|
.form input {
|
||||||
|
height: 38px;
|
||||||
|
padding: 0 10px;
|
||||||
|
border: 1px solid var(--border);
|
||||||
|
border-radius: 8px;
|
||||||
|
background: var(--bg);
|
||||||
|
color: var(--text);
|
||||||
|
font-size: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.form input:focus {
|
||||||
|
outline: 2px solid var(--accent-soft);
|
||||||
|
border-color: var(--accent);
|
||||||
|
}
|
||||||
|
|
||||||
|
.form .button {
|
||||||
|
justify-content: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.form-switch {
|
||||||
|
margin: 4px 0 0;
|
||||||
|
font-size: 13px;
|
||||||
|
color: var(--muted);
|
||||||
|
text-align: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.form-error {
|
||||||
|
margin: 0;
|
||||||
|
color: var(--bad);
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.api-key {
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
align-items: center;
|
||||||
|
margin-bottom: 18px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.api-key code {
|
||||||
|
flex: 1;
|
||||||
|
min-width: 0;
|
||||||
|
padding: 8px 10px;
|
||||||
|
border-radius: 8px;
|
||||||
|
background: var(--bg);
|
||||||
|
font-family: var(--mono);
|
||||||
|
font-size: 12.5px;
|
||||||
|
overflow-wrap: anywhere;
|
||||||
|
}
|
||||||
|
|
||||||
|
.meter {
|
||||||
|
height: 6px;
|
||||||
|
margin-top: 4px;
|
||||||
|
border-radius: 3px;
|
||||||
|
background: var(--border);
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.meter span {
|
||||||
|
display: block;
|
||||||
|
height: 100%;
|
||||||
|
background: var(--accent);
|
||||||
|
}
|
||||||
|
|
||||||
|
.meter-full span {
|
||||||
|
background: var(--bad);
|
||||||
|
}
|
||||||
+189
-7
@@ -51,6 +51,26 @@ const state = {
|
|||||||
let requestId = 0
|
let requestId = 0
|
||||||
let searchAbort = null
|
let searchAbort = null
|
||||||
|
|
||||||
|
/** Signing in (GET /api/v1/access): `open` needs none; with `accounts` the catalog takes a token. */
|
||||||
|
const account = {
|
||||||
|
mode: 'open',
|
||||||
|
signup: 'open',
|
||||||
|
token: storage('get', 'token'),
|
||||||
|
/** The profile's access: tier, limits, today's usage, the Newznab key. */
|
||||||
|
access: null,
|
||||||
|
}
|
||||||
|
|
||||||
|
/** localStorage, which a private window may refuse. */
|
||||||
|
function storage(action, key, value) {
|
||||||
|
try {
|
||||||
|
if (action === 'get') return localStorage.getItem(key)
|
||||||
|
if (action === 'set') localStorage.setItem(key, value)
|
||||||
|
else localStorage.removeItem(key)
|
||||||
|
} catch {
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// --- Formatting ------------------------------------------------------------
|
// --- Formatting ------------------------------------------------------------
|
||||||
|
|
||||||
function formatSize(bytes) {
|
function formatSize(bytes) {
|
||||||
@@ -147,12 +167,26 @@ function syncControls() {
|
|||||||
|
|
||||||
// --- API -------------------------------------------------------------------
|
// --- API -------------------------------------------------------------------
|
||||||
|
|
||||||
async function api(path, { signal } = {}) {
|
async function api(path, { signal, method = 'GET', body } = {}) {
|
||||||
const response = await fetch(path, { headers: { accept: 'application/json' }, signal })
|
const headers = { accept: 'application/json' }
|
||||||
if (!response.ok) throw new Error(`${response.status} ${response.statusText}`)
|
if (account.token) headers.authorization = `Bearer ${account.token}`
|
||||||
|
if (body) headers['content-type'] = 'application/json'
|
||||||
|
const response = await fetch(path, { method, headers, signal, body: body && JSON.stringify(body) })
|
||||||
|
if (!response.ok) {
|
||||||
|
const error = await response.json().catch(() => ({}))
|
||||||
|
// A token that no longer works: sign in again.
|
||||||
|
if (response.status === 401 && account.mode === 'accounts' && !path.startsWith('/api/v1/auth/')) signOut()
|
||||||
|
throw new Error(error.error ?? error.errors?.[0]?.message ?? `${response.status} ${response.statusText}`)
|
||||||
|
}
|
||||||
return response.json()
|
return response.json()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** A release's NZB as a plain link: with accounts it carries the key (a link can't send the token). */
|
||||||
|
function nzbHref(id) {
|
||||||
|
const key = account.mode === 'accounts' && account.access?.apiKey
|
||||||
|
return `/api/v1/releases/${id}/nzb${key ? `?apikey=${encodeURIComponent(key)}` : ''}`
|
||||||
|
}
|
||||||
|
|
||||||
function searchParams(page) {
|
function searchParams(page) {
|
||||||
const params = new URLSearchParams({ sort: state.sort, page: String(page), per_page: '50' })
|
const params = new URLSearchParams({ sort: state.sort, page: String(page), per_page: '50' })
|
||||||
if (state.q) params.set('q', state.q)
|
if (state.q) params.set('q', state.q)
|
||||||
@@ -229,7 +263,7 @@ function releaseRow(release) {
|
|||||||
|
|
||||||
row.querySelector('.release-size').textContent = formatSize(release.size)
|
row.querySelector('.release-size').textContent = formatSize(release.size)
|
||||||
const nzb = row.querySelector('.nzb')
|
const nzb = row.querySelector('.nzb')
|
||||||
nzb.href = `/api/v1/releases/${release.id}/nzb`
|
nzb.href = nzbHref(release.id)
|
||||||
row.querySelector('.release-main').addEventListener('click', () => openRelease(release.id, true))
|
row.querySelector('.release-main').addEventListener('click', () => openRelease(release.id, true))
|
||||||
return row
|
return row
|
||||||
}
|
}
|
||||||
@@ -264,6 +298,8 @@ async function loadResults({ append = false } = {}) {
|
|||||||
(shown < result.total ? ` · showing ${number.format(shown)}` : '')
|
(shown < result.total ? ` · showing ${number.format(shown)}` : '')
|
||||||
$('#empty').hidden = result.total !== 0
|
$('#empty').hidden = result.total !== 0
|
||||||
$('#more').hidden = shown >= result.total && !result.totalCapped
|
$('#more').hidden = shown >= result.total && !result.totalCapped
|
||||||
|
// The day's count in the account button (the profile isn't counted).
|
||||||
|
if (account.access) loadProfile().catch(() => {})
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
if (id !== requestId) return
|
if (id !== requestId) return
|
||||||
$('#list').classList.remove('stale')
|
$('#list').classList.remove('stale')
|
||||||
@@ -299,7 +335,7 @@ function renderDetail(release) {
|
|||||||
const actions = el(
|
const actions = el(
|
||||||
'div',
|
'div',
|
||||||
{ class: 'actions' },
|
{ class: 'actions' },
|
||||||
el('a', { class: 'button button-primary', href: `/api/v1/releases/${release.id}/nzb` },
|
el('a', { class: 'button button-primary', href: nzbHref(release.id) },
|
||||||
el('span', { text: 'Download NZB' })),
|
el('span', { text: 'Download NZB' })),
|
||||||
el('button', {
|
el('button', {
|
||||||
class: 'button',
|
class: 'button',
|
||||||
@@ -418,8 +454,151 @@ function closeRelease(push = false) {
|
|||||||
document.querySelector(`.release[data-id="${id}"] .release-main`)?.focus()
|
document.querySelector(`.release[data-id="${id}"] .release-main`)?.focus()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- Account (ACCESS=accounts) ---------------------------------------------
|
||||||
|
|
||||||
|
function renderAccountButton() {
|
||||||
|
const button = $('#account')
|
||||||
|
button.hidden = account.mode !== 'accounts'
|
||||||
|
if (!account.access) return button.replaceChildren('Sign in')
|
||||||
|
const { tier, usage, limits } = account.access
|
||||||
|
const of = (used, max) => (max === null ? number.format(used) : `${number.format(used)}/${number.format(max)}`)
|
||||||
|
button.title = 'Searches and NZBs today'
|
||||||
|
button.replaceChildren(el('span', { class: 'tier', text: tier }), ` ${of(usage.apiHits, limits.apiHits)} · ${of(usage.grabs, limits.grabs)} NZB`)
|
||||||
|
}
|
||||||
|
|
||||||
|
async function loadProfile() {
|
||||||
|
const { data } = await api('/api/v1/account/profile')
|
||||||
|
account.access = data.access
|
||||||
|
renderAccountButton()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Signed in with a new token: the profile, then the search that was waiting. */
|
||||||
|
async function signedIn(token) {
|
||||||
|
account.token = token
|
||||||
|
storage('set', 'token', token)
|
||||||
|
await loadProfile()
|
||||||
|
$('#account-dialog').close()
|
||||||
|
loadResults()
|
||||||
|
if (state.release) openRelease(state.release)
|
||||||
|
}
|
||||||
|
|
||||||
|
function signOut() {
|
||||||
|
if (account.token) api('/api/v1/account/logout', { method: 'POST' }).catch(() => {})
|
||||||
|
account.token = null
|
||||||
|
account.access = null
|
||||||
|
storage('remove', 'token')
|
||||||
|
renderAccountButton()
|
||||||
|
closeRelease()
|
||||||
|
$('#list').replaceChildren()
|
||||||
|
$('#more').hidden = true
|
||||||
|
$('#empty').hidden = true
|
||||||
|
$('#summary').textContent = 'Sign in to search the index.'
|
||||||
|
openAccount('signin')
|
||||||
|
}
|
||||||
|
|
||||||
|
function field(label, props) {
|
||||||
|
return el('label', {}, label, el('input', props))
|
||||||
|
}
|
||||||
|
|
||||||
|
function accountForm(view) {
|
||||||
|
const signup = view === 'signup'
|
||||||
|
const error = el('p', { class: 'form-error', role: 'alert', hidden: true })
|
||||||
|
const form = el(
|
||||||
|
'form',
|
||||||
|
{ class: 'form' },
|
||||||
|
field('Email', { name: 'email', type: 'email', autocomplete: 'email', required: true }),
|
||||||
|
field('Password', { name: 'password', type: 'password', autocomplete: signup ? 'new-password' : 'current-password', required: true, minlength: signup ? '8' : null }),
|
||||||
|
signup ? field('Password again', { name: 'passwordConfirmation', type: 'password', autocomplete: 'new-password', required: true }) : null,
|
||||||
|
signup && account.signup === 'invite' ? field('Invite code', { name: 'invite', autocomplete: 'off', spellcheck: 'false', required: true }) : null,
|
||||||
|
error,
|
||||||
|
el('button', { class: 'button button-primary', type: 'submit', text: signup ? 'Create account' : 'Sign in' })
|
||||||
|
)
|
||||||
|
form.addEventListener('submit', async (event) => {
|
||||||
|
event.preventDefault()
|
||||||
|
const values = Object.fromEntries(new FormData(form))
|
||||||
|
error.hidden = true
|
||||||
|
try {
|
||||||
|
const path = signup ? '/api/v1/auth/signup' : '/api/v1/auth/login'
|
||||||
|
const { data } = await api(path, { method: 'POST', body: signup ? { fullName: null, ...values } : values })
|
||||||
|
await signedIn(data.token)
|
||||||
|
} catch (failure) {
|
||||||
|
error.textContent = failure.message
|
||||||
|
error.hidden = false
|
||||||
|
}
|
||||||
|
})
|
||||||
|
const other = signup
|
||||||
|
? el('p', { class: 'form-switch' }, 'Have an account? ', el('button', { class: 'link-button', type: 'button', text: 'Sign in', onclick: () => openAccount('signin') }))
|
||||||
|
: account.signup !== 'closed'
|
||||||
|
? el('p', { class: 'form-switch' }, 'New here? ', el('button', { class: 'link-button', type: 'button', text: 'Create an account', onclick: () => openAccount('signup') }))
|
||||||
|
: null
|
||||||
|
return [form, other]
|
||||||
|
}
|
||||||
|
|
||||||
|
function meter(used, max) {
|
||||||
|
if (max === null) return null
|
||||||
|
const share = Math.min(1, used / Math.max(1, max))
|
||||||
|
return el('div', { class: `meter${used >= max ? ' meter-full' : ''}` }, el('span', { style: `width: ${Math.round(share * 100)}%` }))
|
||||||
|
}
|
||||||
|
|
||||||
|
function profileView() {
|
||||||
|
const { tier, tierUntil, limits, usage, apiKey } = account.access
|
||||||
|
const of = (used, max) => (max === null ? `${number.format(used)} (no limit)` : `${number.format(used)} of ${number.format(max)}`)
|
||||||
|
const key = el('code', { text: apiKey })
|
||||||
|
return [
|
||||||
|
el('dl', { class: 'facts' },
|
||||||
|
fact('Tier', tierUntil ? `${tier} until ${new Date(tierUntil).toLocaleDateString()}` : tier),
|
||||||
|
fact('Searches today', el('div', {}, of(usage.apiHits, limits.apiHits), meter(usage.apiHits, limits.apiHits))),
|
||||||
|
fact('NZBs today', el('div', {}, of(usage.grabs, limits.grabs), meter(usage.grabs, limits.grabs))),
|
||||||
|
fact('Retention', limits.retentionDays ? `${number.format(limits.retentionDays)} days` : 'Everything')),
|
||||||
|
...section('Newznab API key', 'for Prowlarr, Sonarr, Radarr', el('div', { class: 'api-key' },
|
||||||
|
key,
|
||||||
|
el('button', { class: 'button', type: 'button', text: 'Copy', onclick: async (event) => {
|
||||||
|
await navigator.clipboard?.writeText(account.access.apiKey)
|
||||||
|
event.target.textContent = 'Copied'
|
||||||
|
setTimeout(() => (event.target.textContent = 'Copy'), 1200)
|
||||||
|
} }),
|
||||||
|
el('button', { class: 'button', type: 'button', text: 'New key', title: 'The old key stops working', onclick: async () => {
|
||||||
|
if (!confirm('Make a new key? Apps using the old one stop working until they get the new one.')) return
|
||||||
|
const { apiKey: fresh } = await api('/api/v1/account/api-key', { method: 'POST' })
|
||||||
|
account.access.apiKey = fresh
|
||||||
|
key.textContent = fresh
|
||||||
|
} }))),
|
||||||
|
el('p', { class: 'form-switch' }, `URL for your apps: ${location.origin}`),
|
||||||
|
el('button', { class: 'button', type: 'button', text: 'Sign out', onclick: () => signOut() }),
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openAccount(view) {
|
||||||
|
const dialog = $('#account-dialog')
|
||||||
|
if (view === 'profile') await loadProfile().catch(() => {})
|
||||||
|
if (view === 'profile' && !account.access) return
|
||||||
|
$('#account-title').textContent = { signin: 'Sign in', signup: 'Create an account', profile: 'Your account' }[view]
|
||||||
|
$('#account-body').replaceChildren(...(view === 'profile' ? profileView() : accountForm(view)))
|
||||||
|
if (!dialog.open) dialog.showModal()
|
||||||
|
dialog.querySelector('input')?.focus()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** With accounts, nothing is searched before signing in. */
|
||||||
|
async function startAccess() {
|
||||||
|
try {
|
||||||
|
Object.assign(account, await api('/api/v1/access'))
|
||||||
|
} catch {
|
||||||
|
// An older server: open.
|
||||||
|
}
|
||||||
|
if (account.mode !== 'accounts') return true
|
||||||
|
if (account.token) await loadProfile().catch(() => {})
|
||||||
|
renderAccountButton()
|
||||||
|
if (account.access) return true
|
||||||
|
$('#summary').textContent = 'Sign in to search the index.'
|
||||||
|
if (!$('#account-dialog').open) openAccount('signin')
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// --- Wiring ----------------------------------------------------------------
|
// --- Wiring ----------------------------------------------------------------
|
||||||
|
|
||||||
|
$('#account').addEventListener('click', () => openAccount(account.access ? 'profile' : 'signin'))
|
||||||
|
$('#account-close').addEventListener('click', () => $('#account-dialog').close())
|
||||||
|
|
||||||
$('#q').addEventListener('input', (event) => {
|
$('#q').addEventListener('input', (event) => {
|
||||||
state.q = event.target.value.trim()
|
state.q = event.target.value.trim()
|
||||||
scheduleSearch()
|
scheduleSearch()
|
||||||
@@ -495,5 +674,8 @@ window.addEventListener('popstate', () => {
|
|||||||
readUrl()
|
readUrl()
|
||||||
syncControls()
|
syncControls()
|
||||||
loadStats()
|
loadStats()
|
||||||
loadResults()
|
startAccess().then((ready) => {
|
||||||
if (state.release) openRelease(state.release)
|
if (!ready) return
|
||||||
|
loadResults()
|
||||||
|
if (state.release) openRelease(state.release)
|
||||||
|
})
|
||||||
@@ -14,8 +14,19 @@
|
|||||||
<span>Voyager 2</span>
|
<span>Voyager 2</span>
|
||||||
</a>
|
</a>
|
||||||
<dl class="stats" id="stats" aria-label="Index statistics"></dl>
|
<dl class="stats" id="stats" aria-label="Index statistics"></dl>
|
||||||
|
<button type="button" class="button account-button" id="account" hidden></button>
|
||||||
</header>
|
</header>
|
||||||
|
|
||||||
|
<dialog class="dialog" id="account-dialog" aria-labelledby="account-title">
|
||||||
|
<div class="dialog-head">
|
||||||
|
<h2 id="account-title"></h2>
|
||||||
|
<button type="button" class="icon-button" id="account-close" aria-label="Close">
|
||||||
|
<svg aria-hidden="true" viewBox="0 0 24 24"><path d="M6 6l12 12M18 6 6 18" /></svg>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<div class="dialog-body" id="account-body"></div>
|
||||||
|
</dialog>
|
||||||
|
|
||||||
<main class="layout">
|
<main class="layout">
|
||||||
<section class="search" aria-label="Search">
|
<section class="search" aria-label="Search">
|
||||||
<div class="search-row">
|
<div class="search-row">
|
||||||
|
|||||||
@@ -52,6 +52,10 @@ export default await Env.create(new URL('../', import.meta.url), {
|
|||||||
// Newznab API key (empty: no key needed)
|
// Newznab API key (empty: no key needed)
|
||||||
NEWZNAB_API_KEY: Env.schema.secret.optional(),
|
NEWZNAB_API_KEY: Env.schema.secret.optional(),
|
||||||
|
|
||||||
|
// Access (config/access.ts): open (one key, no limits) or accounts (a key per user, tiers with daily limits)
|
||||||
|
ACCESS: Env.schema.enum.optional(['open', 'accounts'] as const),
|
||||||
|
SIGNUP: Env.schema.enum.optional(['open', 'invite', 'closed'] as const),
|
||||||
|
|
||||||
// Coordinator
|
// Coordinator
|
||||||
RANGE_ARTICLES: Env.schema.number.optional(),
|
RANGE_ARTICLES: Env.schema.number.optional(),
|
||||||
TAIL_EVERY_SECONDS: Env.schema.number.optional(),
|
TAIL_EVERY_SECONDS: Env.schema.number.optional(),
|
||||||
|
|||||||
+6
-1
@@ -45,13 +45,18 @@ router
|
|||||||
router
|
router
|
||||||
.group(() => {
|
.group(() => {
|
||||||
router.get('profile', [controllers.Profile, 'show'])
|
router.get('profile', [controllers.Profile, 'show'])
|
||||||
|
router.post('api-key', [controllers.Profile, 'resetApiKey'])
|
||||||
router.post('logout', [controllers.AccessTokens, 'destroy'])
|
router.post('logout', [controllers.AccessTokens, 'destroy'])
|
||||||
})
|
})
|
||||||
.prefix('account')
|
.prefix('account')
|
||||||
.as('profile')
|
.as('profile')
|
||||||
.use(middleware.auth())
|
.use(middleware.auth())
|
||||||
|
|
||||||
// The catalog (REBUILD.md, part 9): open, as voyager 1's (the web interface uses it); Newznab has its key.
|
// How the web interface signs in: open, or accounts with limits (config/access.ts).
|
||||||
|
router.get('access', [controllers.Releases, 'access'])
|
||||||
|
|
||||||
|
// The catalog (REBUILD.md, part 9): open, as voyager 1's (the web interface uses it), unless ACCESS
|
||||||
|
// is `accounts`: then a user's token or apikey, counted as Newznab's are (AccessService).
|
||||||
router.get('releases', [controllers.Releases, 'index'])
|
router.get('releases', [controllers.Releases, 'index'])
|
||||||
router.get('releases/:id', [controllers.Releases, 'show']).where('id', router.matchers.number())
|
router.get('releases/:id', [controllers.Releases, 'show']).where('id', router.matchers.number())
|
||||||
router.get('releases/:id/nzb', [controllers.Releases, 'nzb']).where('id', router.matchers.number())
|
router.get('releases/:id/nzb', [controllers.Releases, 'nzb']).where('id', router.matchers.number())
|
||||||
|
|||||||
@@ -0,0 +1,155 @@
|
|||||||
|
import { test } from '@japa/runner'
|
||||||
|
import { DateTime } from 'luxon'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import testUtils from '@adonisjs/core/services/test_utils'
|
||||||
|
import User from '#models/user'
|
||||||
|
import BatchStore from '#services/batch_store'
|
||||||
|
import AccessService from '#services/access_service'
|
||||||
|
import { TIERS } from '#config/access'
|
||||||
|
import { RangeIndexer } from '#lib/edge/index'
|
||||||
|
import { DEFAULT_RULES } from '#lib/rules/index'
|
||||||
|
import type { Overview } from '#lib/nntp/index'
|
||||||
|
|
||||||
|
const GROUP = 'alt.binaries.test'
|
||||||
|
|
||||||
|
function file(name: string, counter: string, parts: number, at: number): Overview[] {
|
||||||
|
return Array.from({ length: parts }, (_, i) => [at + i, {
|
||||||
|
'subject': `${counter} - "${name}" yEnc (${i + 1}/${parts})`, 'from': 'p <p@x>', 'date': 'Mon, 05 Oct 2026 10:00:00 +0000',
|
||||||
|
'message-id': `<${name}-${i + 1}@test>`, ':bytes': String(700000 + i), 'xref': `news.example ${GROUP}:${at + i}`,
|
||||||
|
}] as Overview)
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Two releases: a show (recent) and a movie posted two years ago. */
|
||||||
|
async function catalog() {
|
||||||
|
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
||||||
|
const [n] = await db.table('nodes').insert({ name: 'access-node' }).returning('id')
|
||||||
|
const [r] = await db.table('ranges').insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: n.id }).returning('id')
|
||||||
|
const indexer = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
|
||||||
|
indexer.add([...file('Some.Show.S01E01.1080p.WEB.mkv', 'Some.Show.S01E01.1080p.WEB [1/1]', 2, 100), ...file('Old.Movie.2024.1080p.BluRay.mkv', 'Old.Movie.2024.1080p.BluRay [1/1]', 1, 300)], 1001)
|
||||||
|
indexer.finish()
|
||||||
|
await BatchStore.store(Number(r.id), Number(n.id), indexer.take())
|
||||||
|
const show = await db.from('releases').where('name', 'ilike', '%show%').firstOrFail()
|
||||||
|
const movie = await db.from('releases').where('name', 'ilike', '%movie%').firstOrFail()
|
||||||
|
await db.from('releases').where('id', movie.id).update({ posted_at: DateTime.now().minus({ years: 2 }).toJSDate() })
|
||||||
|
return { show: Number(show.id), movie: Number(movie.id) }
|
||||||
|
}
|
||||||
|
|
||||||
|
const user = (email = 'someone@example.com') => User.create({ email, password: 'secret-password' })
|
||||||
|
|
||||||
|
test.group('Access with accounts', (group) => {
|
||||||
|
const free = { ...TIERS.free }
|
||||||
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||||
|
group.each.setup(() => {
|
||||||
|
AccessService.mode = 'accounts'
|
||||||
|
AccessService.signup = 'invite'
|
||||||
|
Object.assign(TIERS.free, { apiHits: 3, grabs: 1, retentionDays: 365 })
|
||||||
|
return () => {
|
||||||
|
AccessService.mode = 'open'
|
||||||
|
AccessService.signup = 'open'
|
||||||
|
Object.assign(TIERS.free, free)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
test("Newznab takes a user's key, counts searches and grabs per day and keeps to the tier's retention", async ({ client, assert }) => {
|
||||||
|
const { show, movie } = await catalog()
|
||||||
|
const someone = await user()
|
||||||
|
|
||||||
|
assert.include((await client.get('/api?t=search&q=')).text(), 'code="100"')
|
||||||
|
assert.include((await client.get('/api?t=search&q=&apikey=0123456789abcdef0123456789abcdef')).text(), 'code="100"')
|
||||||
|
// caps isn't counted.
|
||||||
|
assert.include((await client.get(`/api?t=caps&apikey=${someone.apiKey}`)).text(), '<registration available="yes" open="no"/>')
|
||||||
|
|
||||||
|
const feed = (await client.get(`/api?t=search&q=&apikey=${someone.apiKey}`)).text()
|
||||||
|
// The free tier sees a year back: the movie isn't found.
|
||||||
|
assert.include(feed, 'Some Show S01E01')
|
||||||
|
assert.notInclude(feed, 'Old Movie')
|
||||||
|
assert.include(feed, `t=get&id=${show}&apikey=${someone.apiKey}`)
|
||||||
|
assert.include(feed, '<newznab:apilimits apicurrent="1" apimax="3" grabcurrent="0" grabmax="1"/>')
|
||||||
|
|
||||||
|
// Past the retention: refused, and not counted as a grab.
|
||||||
|
assert.include((await client.get(`/api?t=get&id=${movie}&apikey=${someone.apiKey}`)).text(), 'code="102"')
|
||||||
|
assert.include((await client.get(`/api?t=get&id=999999&apikey=${someone.apiKey}`)).text(), 'code="300"')
|
||||||
|
const nzb = await client.get(`/api?t=get&id=${show}&apikey=${someone.apiKey}`)
|
||||||
|
assert.include(nzb.header('content-type'), 'application/x-nzb')
|
||||||
|
assert.include((await client.get(`/api?t=get&id=${show}&apikey=${someone.apiKey}`)).text(), 'code="501"')
|
||||||
|
|
||||||
|
await client.get(`/api?t=search&q=&apikey=${someone.apiKey}`)
|
||||||
|
await client.get(`/api?t=search&q=&apikey=${someone.apiKey}`)
|
||||||
|
assert.include((await client.get(`/api?t=search&q=&apikey=${someone.apiKey}`)).text(), 'code="500"')
|
||||||
|
assert.deepEqual(await AccessService.usage(someone.id), { apiHits: 3, grabs: 1 })
|
||||||
|
|
||||||
|
// VIP sees everything; a tier that ran out is free again.
|
||||||
|
AccessService.grantTier(someone, 'vip', 30)
|
||||||
|
await someone.save()
|
||||||
|
assert.include((await client.get(`/api?t=movie&q=old movie&apikey=${someone.apiKey}`)).text(), 'Old Movie 2024')
|
||||||
|
someone.tierUntil = DateTime.now().minus({ days: 1 })
|
||||||
|
await someone.save()
|
||||||
|
assert.include((await client.get(`/api?t=movie&q=old movie&apikey=${someone.apiKey}`)).text(), 'code="500"')
|
||||||
|
|
||||||
|
someone.suspended = true
|
||||||
|
await someone.save()
|
||||||
|
assert.include((await client.get(`/api?t=caps&apikey=${someone.apiKey}`)).text(), 'code="101"')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('the catalog needs a token or a key, and counts as Newznab does', async ({ client, assert }) => {
|
||||||
|
const { show, movie } = await catalog()
|
||||||
|
const someone = await user()
|
||||||
|
|
||||||
|
;(await client.get('/api/v1/releases')).assertStatus(401)
|
||||||
|
;(await client.get(`/api/v1/releases/${show}/nzb`)).assertStatus(401)
|
||||||
|
// Stats and how to sign in stay open.
|
||||||
|
;(await client.get('/api/v1/stats')).assertStatus(200)
|
||||||
|
assert.deepEqual((await client.get('/api/v1/access')).body(), { mode: 'accounts', signup: 'invite' })
|
||||||
|
|
||||||
|
const found = await client.get('/api/v1/releases').loginAs(someone)
|
||||||
|
found.assertStatus(200)
|
||||||
|
assert.deepEqual((found.body() as { releases: { id: number }[] }).releases.map((r) => r.id), [show])
|
||||||
|
;(await client.get(`/api/v1/releases/${movie}`).loginAs(someone)).assertStatus(403)
|
||||||
|
;(await client.get(`/api/v1/releases/${show}`).loginAs(someone)).assertStatus(200)
|
||||||
|
|
||||||
|
// A plain link (the web interface's download button) carries the key.
|
||||||
|
const nzb = await client.get(`/api/v1/releases/${show}/nzb?apikey=${someone.apiKey}`)
|
||||||
|
nzb.assertStatus(200)
|
||||||
|
const over = await client.get(`/api/v1/releases/${show}/nzb?apikey=${someone.apiKey}`)
|
||||||
|
over.assertStatus(429)
|
||||||
|
assert.deepEqual(over.body(), { error: 'Download limit reached', code: 501 })
|
||||||
|
// The refused movie wasn't counted: a third search is left.
|
||||||
|
;(await client.get('/api/v1/releases').loginAs(someone)).assertStatus(200)
|
||||||
|
;(await client.get('/api/v1/releases').loginAs(someone)).assertStatus(429)
|
||||||
|
|
||||||
|
const profile = ((await client.get('/api/v1/account/profile').loginAs(someone)).body() as any).data
|
||||||
|
assert.equal(profile.access.tier, 'free')
|
||||||
|
assert.equal(profile.access.apiKey, someone.apiKey)
|
||||||
|
assert.deepEqual(profile.access.usage, { apiHits: 3, grabs: 1 })
|
||||||
|
|
||||||
|
const reset = (await client.post('/api/v1/account/api-key').loginAs(someone)).body() as { apiKey: string }
|
||||||
|
assert.notEqual(reset.apiKey, someone.apiKey)
|
||||||
|
assert.include((await client.get(`/api?t=caps&apikey=${someone.apiKey}`)).text(), 'code="100"')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('signup takes an unused invite, which may give a tier', async ({ client, assert }) => {
|
||||||
|
const form = (email: string, invite?: string) => ({ email, password: 'secret-password', passwordConfirmation: 'secret-password', fullName: null, invite })
|
||||||
|
await db.table('invites').insert([{ code: 'plain-invite' }, { code: 'vip-invite', tier: 'vip', tier_days: 365 }])
|
||||||
|
|
||||||
|
;(await client.post('/api/v1/auth/signup').json(form('a@example.com'))).assertStatus(403)
|
||||||
|
;(await client.post('/api/v1/auth/signup').json(form('a@example.com', 'no-such-code'))).assertStatus(403)
|
||||||
|
// A refused invite leaves no account behind.
|
||||||
|
assert.isNull(await User.findBy('email', 'a@example.com'))
|
||||||
|
|
||||||
|
;(await client.post('/api/v1/auth/signup').json(form('a@example.com', 'plain-invite'))).assertStatus(200)
|
||||||
|
;(await client.post('/api/v1/auth/signup').json(form('b@example.com', 'plain-invite'))).assertStatus(403)
|
||||||
|
const vip = await client.post('/api/v1/auth/signup').json(form('c@example.com', 'vip-invite'))
|
||||||
|
vip.assertStatus(200)
|
||||||
|
const c = await User.findByOrFail('email', 'c@example.com')
|
||||||
|
assert.equal(c.tier, 'vip')
|
||||||
|
assert.closeTo(c.tierUntil!.diffNow('days').days, 365, 1)
|
||||||
|
assert.match(c.apiKey, /^[0-9a-f]{32}$/)
|
||||||
|
|
||||||
|
// Renewed early, the year is added to what's left.
|
||||||
|
AccessService.grantTier(c, 'vip', 365)
|
||||||
|
assert.closeTo(c.tierUntil!.diffNow('days').days, 730, 1)
|
||||||
|
|
||||||
|
AccessService.signup = 'closed'
|
||||||
|
;(await client.post('/api/v1/auth/signup').json(form('d@example.com', 'vip-invite'))).assertStatus(403)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import { test } from '@japa/runner'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import testUtils from '@adonisjs/core/services/test_utils'
|
||||||
|
import BatchStore from '#services/batch_store'
|
||||||
|
import RangeBlocks from '#services/range_blocks'
|
||||||
|
import SearchService from '#services/search_service'
|
||||||
|
import { RangeIndexer, postedSpan } from '#lib/edge/index'
|
||||||
|
import { DEFAULT_RULES } from '#lib/rules/index'
|
||||||
|
import type { Overview } from '#lib/nntp/index'
|
||||||
|
|
||||||
|
const GROUP = 'alt.binaries.final'
|
||||||
|
const at = (iso: string) => new Date(iso)
|
||||||
|
|
||||||
|
test.group('Final completeness', (group) => {
|
||||||
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||||
|
|
||||||
|
test("a batch's posting times, without the earliest and latest 1%, become the range's span", async ({ assert }) => {
|
||||||
|
assert.deepEqual(postedSpan([]), [null, null])
|
||||||
|
const dates = Array.from({ length: 200 }, (_, i) => 1_000_000 + i)
|
||||||
|
dates.push(5, 9_999_999_999)
|
||||||
|
// 202 dates: 2 trimmed at each end, so the two wild ones go.
|
||||||
|
assert.deepEqual(postedSpan(dates), [1_000_001, 1_000_198])
|
||||||
|
|
||||||
|
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
||||||
|
const [node] = await db.table('nodes').insert({ name: 'final-node' }).returning('id')
|
||||||
|
const [range] = await db
|
||||||
|
.table('ranges')
|
||||||
|
.insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: node.id, rule_version: 1 })
|
||||||
|
.returning('id')
|
||||||
|
const overviews = Array.from({ length: 10 }, (_, i) => {
|
||||||
|
const date = new Date(Date.parse('2026-01-01T10:00:00Z') + i * 60_000).toUTCString().replace('GMT', '+0000')
|
||||||
|
return [1 + i, { 'subject': `[1/1] - "Some.Thing.2026.rar" yEnc (${i + 1}/10)`, 'from': 'p <p@x>', 'date': date, 'message-id': `<f${i}@x>`, ':bytes': '1000', 'xref': `x ${GROUP}:${1 + i}` }] as Overview
|
||||||
|
})
|
||||||
|
const ix = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
|
||||||
|
ix.add(overviews, 11)
|
||||||
|
await BatchStore.store(Number(range.id), Number(node.id), ix.take())
|
||||||
|
const stored = await db.from('ranges').where('id', range.id).firstOrFail()
|
||||||
|
assert.equal(new Date(stored.posted_low).toISOString(), '2026-01-01T10:00:00.000Z')
|
||||||
|
assert.equal(new Date(stored.posted_high).toISOString(), '2026-01-01T10:09:00.000Z')
|
||||||
|
})
|
||||||
|
|
||||||
|
test('a release is final once one block of done ranges covers its posting times and 6 hours either side', async ({ assert }) => {
|
||||||
|
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
||||||
|
const range = (start: number, status: string, low: string | null, high: string | null) =>
|
||||||
|
db.table('ranges').insert({ group_id: g.id, start_id: start, end_id: start + 99_999, next_id: start, status, posted_low: low && at(low), posted_high: high && at(high) })
|
||||||
|
await range(1, 'done', '2026-01-01T00:00:00Z', '2026-01-01T12:00:00Z')
|
||||||
|
await range(100_001, 'done', '2026-01-01T12:00:00Z', '2026-01-01T23:00:00Z')
|
||||||
|
await range(200_001, 'planned', null, null)
|
||||||
|
await range(300_001, 'done', '2026-01-02T10:00:00Z', '2026-01-02T20:00:00Z')
|
||||||
|
assert.equal(await RangeBlocks.rebuild(), 2)
|
||||||
|
|
||||||
|
let n = 0
|
||||||
|
const release = async (first: number, posted: string, extra: Record<string, unknown> = {}) => {
|
||||||
|
n++
|
||||||
|
const key = `fc${n}`.padEnd(40, '0')
|
||||||
|
const [row] = await db
|
||||||
|
.table('releases')
|
||||||
|
.insert({ release_key: key, identity_key: key, name: `Final.Test.${n}`, source: 'stem', groups: '[]', group_id: g.id, files: 3, file_total: 5, complete: false, first_article: first, last_article: first + 10, posted_at: at(posted), last_posted_at: at(posted), readable: true, settled_at: new Date(), ...extra })
|
||||||
|
.returning('id')
|
||||||
|
return Number(row.id)
|
||||||
|
}
|
||||||
|
const final = async (id: number) => {
|
||||||
|
const r = await db.from('releases').where('id', id).firstOrFail()
|
||||||
|
const { rows } = await db.rawQuery(`SELECT release_final(?, ?, ?, ?, ?, ?) AS final`, [r.group_id, r.first_article, r.last_article, r.posted_at, r.last_posted_at, r.complete])
|
||||||
|
return rows[0].final as boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mid-block: final either way.
|
||||||
|
const middle = await release(90_000, '2026-01-01T11:00:00Z')
|
||||||
|
// In the first range: nothing older to wait for, although 6 hours before it isn't covered.
|
||||||
|
const first = await release(1_000, '2026-01-01T02:00:00Z')
|
||||||
|
// Near the block's end, the next range unread: its missing files may be there.
|
||||||
|
const edge = await release(195_000, '2026-01-01T22:00:00Z')
|
||||||
|
// In the island after the gap: the gap is before it.
|
||||||
|
const island = await release(310_000, '2026-01-02T12:00:00Z')
|
||||||
|
assert.isTrue(await final(middle))
|
||||||
|
assert.isTrue(await final(first))
|
||||||
|
assert.isFalse(await final(edge))
|
||||||
|
// Complete there: every file found, nothing to wait for.
|
||||||
|
assert.isTrue(await final(await release(196_000, '2026-01-01T22:30:00Z', { files: 5, complete: true, name: 'Complete.Edge' })))
|
||||||
|
assert.isFalse(await final(island))
|
||||||
|
|
||||||
|
// Search: an incomplete release that isn't final isn't left out as incomplete; the API says which is final.
|
||||||
|
const shown = await SearchService.search({ q: 'Final.Test', notIncomplete: true })
|
||||||
|
assert.sameMembers(shown.releases.map((r) => r.id), [edge, island])
|
||||||
|
assert.isTrue(shown.releases.every((r) => r.final === false))
|
||||||
|
|
||||||
|
// The gap read: one block, the edge release final (the island's 12:00 + 6 h is inside 20:00).
|
||||||
|
await db.from('ranges').where('start_id', 200_001).update({ status: 'done', posted_low: at('2026-01-01T23:00:00Z'), posted_high: at('2026-01-02T10:00:00Z') })
|
||||||
|
assert.equal(await RangeBlocks.rebuild(), 1)
|
||||||
|
assert.isTrue(await final(edge))
|
||||||
|
assert.isTrue(await final(island))
|
||||||
|
assert.equal((await SearchService.search({ q: 'Final.Test', notIncomplete: true })).total, 0)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
import { test } from '@japa/runner'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import testUtils from '@adonisjs/core/services/test_utils'
|
||||||
|
import Leases from '#services/leases'
|
||||||
|
|
||||||
|
test.group('Leases', (group) => {
|
||||||
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||||
|
|
||||||
|
test('at most two ranges of a group are leased at a time; a stale lease is taken over', async ({ assert }) => {
|
||||||
|
const [a] = await db.table('groups').insert({ name: 'alt.binaries.lease.a', priority: 1 }).returning('id')
|
||||||
|
const [b] = await db.table('groups').insert({ name: 'alt.binaries.lease.b', priority: 2 }).returning('id')
|
||||||
|
const range = async (groupId: number, start: number) => {
|
||||||
|
const [row] = await db.table('ranges').insert({ group_id: groupId, start_id: start, end_id: start + 99, next_id: start, priority: -1 }).returning('id')
|
||||||
|
return Number(row.id)
|
||||||
|
}
|
||||||
|
// Four of group a first in id order, then one of b.
|
||||||
|
const a1 = await range(a.id, 1)
|
||||||
|
const a2 = await range(a.id, 101)
|
||||||
|
await range(a.id, 201)
|
||||||
|
await range(a.id, 301)
|
||||||
|
const b1 = await range(b.id, 1)
|
||||||
|
const node = async (name: string) => Number((await db.table('nodes').insert({ name }).returning('id'))[0].id)
|
||||||
|
|
||||||
|
const leased = async (n: number) => (await Leases.lease(await node(`lease-${n}`)))?.range.id ?? null
|
||||||
|
assert.equal(await leased(1), a1)
|
||||||
|
assert.equal(await leased(2), a2)
|
||||||
|
// A third of group a would only wait on the group's lock: group b's.
|
||||||
|
assert.equal(await leased(3), b1)
|
||||||
|
assert.isNull(await leased(4))
|
||||||
|
|
||||||
|
// a1's worker went quiet: its lease is taken over (a2 alone counts).
|
||||||
|
await db.from('ranges').where('id', a1).update({ heartbeat_at: new Date(Date.now() - 24 * 3600_000) })
|
||||||
|
assert.equal(await leased(5), a1)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -2,6 +2,8 @@ import { test } from '@japa/runner'
|
|||||||
import db from '@adonisjs/lucid/services/db'
|
import db from '@adonisjs/lucid/services/db'
|
||||||
import testUtils from '@adonisjs/core/services/test_utils'
|
import testUtils from '@adonisjs/core/services/test_utils'
|
||||||
import { Par2Merger } from '#services/par2_merger'
|
import { Par2Merger } from '#services/par2_merger'
|
||||||
|
import { encodeChunk } from '#lib/chunks/index'
|
||||||
|
import ChunkPartitions from '#services/chunk_partitions'
|
||||||
|
|
||||||
const hash = (c: string) => c.repeat(32)
|
const hash = (c: string) => c.repeat(32)
|
||||||
|
|
||||||
@@ -47,7 +49,7 @@ test.group('Par2Merger', (group) => {
|
|||||||
await db.table('par2_sets').insert({ set_id: setId, release_id: par2, files: 2 })
|
await db.table('par2_sets').insert({ set_id: setId, release_id: par2, files: 2 })
|
||||||
|
|
||||||
const round = await new Par2Merger().round()
|
const round = await new Par2Merger().round()
|
||||||
assert.deepEqual(round, { sets: 1, merges: 1, releases: 3, stems: 0, stemMerges: 0, stemReleases: 0 })
|
assert.deepEqual(round, { sets: 1, merges: 1, releases: 3, stems: 0, stemMerges: 0, stemReleases: 0, folded: { looked: 0, sidecars: 0, folded: 0 } })
|
||||||
const target = await db.from('releases').where('id', b).firstOrFail()
|
const target = await db.from('releases').where('id', b).firstOrFail()
|
||||||
assert.equal(target.files, 4)
|
assert.equal(target.files, 4)
|
||||||
assert.equal(target.segments, 40)
|
assert.equal(target.segments, 40)
|
||||||
@@ -63,6 +65,19 @@ test.group('Par2Merger', (group) => {
|
|||||||
assert.deepEqual(await new Par2Merger().round().then((r) => r.merges), 0)
|
assert.deepEqual(await new Par2Merger().round().then((r) => r.merges), 0)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('a restart goes on from the last whole round, kept in worker_cursors', async ({ assert }) => {
|
||||||
|
const hourAgo = new Date(Date.now() - 3600_000)
|
||||||
|
await db.table('par2_sets').insert({ set_id: 'f'.repeat(32), release_id: 1, files: 2, created_at: hourAgo })
|
||||||
|
// Never run: everything.
|
||||||
|
assert.equal((await new Par2Merger().round()).sets, 1)
|
||||||
|
const saved = await db.from('worker_cursors').where('name', 'merger').firstOrFail()
|
||||||
|
assert.isAbove(new Date(saved.at).getTime(), hourAgo.getTime())
|
||||||
|
// A new process (a deploy) reads where the last round started: the set an hour old is behind it.
|
||||||
|
assert.equal((await new Par2Merger().round()).sets, 0)
|
||||||
|
await db.from('worker_cursors').where('name', 'merger').update({ at: new Date(hourAgo.getTime() - 60_000) })
|
||||||
|
assert.equal((await new Par2Merger().round()).sets, 1)
|
||||||
|
})
|
||||||
|
|
||||||
test('a set held by two of the releases merged is held once by the target', async ({ assert }) => {
|
test('a set held by two of the releases merged is held once by the target', async ({ assert }) => {
|
||||||
const at = new Date('2022-07-08T03:46:00Z')
|
const at = new Date('2022-07-08T03:46:00Z')
|
||||||
const ids: number[] = []
|
const ids: number[] = []
|
||||||
@@ -180,4 +195,40 @@ test.group('Par2Merger', (group) => {
|
|||||||
assert.equal((await new Par2Merger().round()).merges, 0)
|
assert.equal((await new Par2Merger().round()).merges, 0)
|
||||||
assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 3)
|
assert.equal((await db.from('releases').whereIn('id', ids).select('id')).length, 3)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test('an .nzb or .nfo apart from its upload is folded into it', async ({ assert }) => {
|
||||||
|
await ChunkPartitions.ensure()
|
||||||
|
const at = new Date('2022-07-08T03:46:00Z')
|
||||||
|
const poster = 'up <up@load.er>'
|
||||||
|
let n = 0
|
||||||
|
const release = async (name: string, files: number, total: number | null, posted: Date, chunkFiles: string[]) => {
|
||||||
|
n++
|
||||||
|
const key = `ff${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
||||||
|
const [row] = await db
|
||||||
|
.table('releases')
|
||||||
|
.insert({ release_key: key, identity_key: key, name, source: 'stem', groups: '[]', group_id: 1, files, file_total: total, poster, posted_at: posted, readable: true, settled_at: posted })
|
||||||
|
.returning('id')
|
||||||
|
const id = Number(row.id)
|
||||||
|
const payload = encodeChunk(chunkFiles.map((f, i) => ({ name: f, poster, subject: `${name} [${String(i).padStart(2, '0')}/${total ?? 0}] - "${f}" yEnc`, postedAt: posted.getTime(), groups: ['a.b.test'], segments: [[1, `${id}-${i}@x`, 1000]] })))
|
||||||
|
await db.table('release_chunks').insert({ release_id: id, range_id: 1, batch_from: 1, files: chunkFiles.length, segments: chunkFiles.length, payload })
|
||||||
|
await db.table('fold_candidates').insert({ release_id: id }).onConflict('release_id').ignore()
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
const upload = await release('Murdoch Mysteries S8D1', 94, 94, at, ['Murdoch Mysteries S8D1.part01.rar', 'Murdoch Mysteries S8D1.part02.rar', 'Murdoch Mysteries S8D1.part03.rar', 'Murdoch Mysteries S8D1.part04.rar'])
|
||||||
|
const other = await release('Another Show S1D1', 50, 50, new Date(at.getTime() + 120_000), ['Another Show S1D1.part01.rar', 'Another Show S1D1.part02.rar', 'Another Show S1D1.part03.rar', 'Another Show S1D1.part04.rar'])
|
||||||
|
const nzb = await release('Murdoch Mysteries S8D1', 1, 80, new Date(at.getTime() + 60_000), ['Murdoch Mysteries S8D1.nzb'])
|
||||||
|
// A picture isn't a sidecar; an nfo an hour later belongs to nothing here.
|
||||||
|
const picture = await release('cover', 1, null, at, ['cover.jpg'])
|
||||||
|
const late = await release('Murdoch Mysteries S8D1', 1, null, new Date(at.getTime() + 3600_000), ['Murdoch Mysteries S8D1.nfo'])
|
||||||
|
|
||||||
|
const round = await new Par2Merger().round()
|
||||||
|
assert.deepEqual(round.folded, { looked: 5, sidecars: 2, folded: 1 })
|
||||||
|
assert.isNull(await db.from('releases').where('id', nzb).first())
|
||||||
|
assert.equal((await db.from('releases').where('id', upload).firstOrFail()).files, 95)
|
||||||
|
assert.equal((await db.from('releases').where('id', other).firstOrFail()).files, 50)
|
||||||
|
assert.isNotNull(await db.from('releases').where('id', picture).first())
|
||||||
|
assert.isNotNull(await db.from('releases').where('id', late).first())
|
||||||
|
assert.equal((await db.from('release_merges').where({ into_id: upload, via: 'sidecar' }).count('* as c'))[0].c, '1')
|
||||||
|
assert.equal((await db.from('fold_candidates').count('* as c'))[0].c, '0')
|
||||||
|
})
|
||||||
})
|
})
|
||||||
@@ -0,0 +1,68 @@
|
|||||||
|
import { test } from '@japa/runner'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import testUtils from '@adonisjs/core/services/test_utils'
|
||||||
|
import { Par2Pruner } from '#services/par2_pruner'
|
||||||
|
|
||||||
|
test.group('Par2Pruner', (group) => {
|
||||||
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||||
|
|
||||||
|
test('drops the file lists of complete and judged sets, keeps the rest', async ({ assert }) => {
|
||||||
|
const old = new Date(Date.now() - 3 * 86400_000)
|
||||||
|
let n = 0
|
||||||
|
const release = async () => {
|
||||||
|
n++
|
||||||
|
const key = `pp${String(n).padStart(6, '0')}`.padEnd(40, '0')
|
||||||
|
const [row] = await db
|
||||||
|
.table('releases')
|
||||||
|
.insert({ release_key: key, identity_key: key, name: key, source: 'stem', groups: '[]', group_id: 1, files: 2, readable: false })
|
||||||
|
.returning('id')
|
||||||
|
return Number(row.id)
|
||||||
|
}
|
||||||
|
let candidate = 5000
|
||||||
|
const set = async (letter: string, holder: number, files: number, hashedIn: number | null, created = old) => {
|
||||||
|
const setId = letter.repeat(32)
|
||||||
|
for (let i = 0; i < files; i++) {
|
||||||
|
const h = `${letter}${i}`.padEnd(32, '0')
|
||||||
|
await db.table('par2_files').insert({ set_id: setId, file_id: h, hash16k: h, length: 1000 + i, name: `f${i}.rar` })
|
||||||
|
if (hashedIn && i < files) {
|
||||||
|
candidate++
|
||||||
|
await db.table('probe_candidates').insert({ id: candidate, release_id: hashedIn, kind: 'lone', message_id: `<${candidate}@x>`, file: 'x', status: 'done' })
|
||||||
|
await db.table('file_hashes').insert({ candidate_id: candidate, release_id: hashedIn, hash16k: h, size: 1000 + i })
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await db.table('par2_sets').insert({ set_id: setId, release_id: holder, files, created_at: created })
|
||||||
|
return setId
|
||||||
|
}
|
||||||
|
const count = async (setId: string) => Number((await db.from('par2_files').where('set_id', setId).count('* as n'))[0].n)
|
||||||
|
|
||||||
|
// Every file's hash in the holder: complete.
|
||||||
|
const a = await release()
|
||||||
|
const complete = await set('a', a, 2, a)
|
||||||
|
// The files hashed in another release (not merged): kept.
|
||||||
|
const b = await release()
|
||||||
|
const apart = await set('b', b, 2, await release())
|
||||||
|
// Complete but recent: kept (its merges may come).
|
||||||
|
const c = await release()
|
||||||
|
const recent = await set('c', c, 2, c, new Date())
|
||||||
|
// Nothing hashed, its only holder dead: judged.
|
||||||
|
const d = await release()
|
||||||
|
const dead = await set('d', d, 2, null, new Date())
|
||||||
|
await db.table('release_verdicts').insert({ release_id: d, rule: 'dead', rule_version: 1 })
|
||||||
|
// Locked isn't junk: kept.
|
||||||
|
const e = await release()
|
||||||
|
const locked = await set('e', e, 2, null)
|
||||||
|
await db.table('release_verdicts').insert({ release_id: e, rule: 'locked', rule_version: 1 })
|
||||||
|
|
||||||
|
const round = await new Par2Pruner().round()
|
||||||
|
assert.deepEqual(round, { complete: 1, judged: 1, rows: 4 })
|
||||||
|
assert.equal(await count(complete), 0)
|
||||||
|
assert.equal(await count(apart), 2)
|
||||||
|
assert.equal(await count(recent), 2)
|
||||||
|
assert.equal(await count(dead), 0)
|
||||||
|
assert.equal(await count(locked), 2)
|
||||||
|
assert.equal((await db.from('par2_sets').count('* as n'))[0].n, '5')
|
||||||
|
|
||||||
|
// Again: the cursor is past them, nothing more.
|
||||||
|
assert.deepEqual(await new Par2Pruner().round(), { complete: 0, judged: 0, rows: 0 })
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -0,0 +1,44 @@
|
|||||||
|
import { test } from '@japa/runner'
|
||||||
|
import db from '@adonisjs/lucid/services/db'
|
||||||
|
import testUtils from '@adonisjs/core/services/test_utils'
|
||||||
|
import BatchStore from '#services/batch_store'
|
||||||
|
import SearchService from '#services/search_service'
|
||||||
|
import { RangeIndexer } from '#lib/edge/index'
|
||||||
|
import { DEFAULT_RULES } from '#lib/rules/index'
|
||||||
|
import type { Overview } from '#lib/nntp/index'
|
||||||
|
|
||||||
|
const GROUP = 'alt.binaries.posters'
|
||||||
|
|
||||||
|
test.group('Poster search', (group) => {
|
||||||
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||||
|
|
||||||
|
test('the batch store keeps each poster once; a poster search finds releases by part of the name', async ({ assert }) => {
|
||||||
|
const [g] = await db.table('groups').insert({ name: GROUP, priority: 1 }).returning('id')
|
||||||
|
const [node] = await db.table('nodes').insert({ name: 'poster-node' }).returning('id')
|
||||||
|
const [range] = await db
|
||||||
|
.table('ranges')
|
||||||
|
.insert({ group_id: g.id, start_id: 1, end_id: 1000, next_id: 1, status: 'leased', node_id: node.id, rule_version: 1 })
|
||||||
|
.returning('id')
|
||||||
|
const post = (n: number, from: string, name: string): Overview => [
|
||||||
|
n,
|
||||||
|
{ 'subject': `[1/1] - "${name}" yEnc (1/1)`, 'from': from, 'date': 'Mon, 05 Oct 2026 10:00:00 +0000', 'message-id': `<ps${n}@x>`, ':bytes': '1000', 'xref': `x ${GROUP}:${n}` },
|
||||||
|
]
|
||||||
|
const ix = new RangeIndexer({ group: GROUP, start: 1, end: 1000, rules: { ...DEFAULT_RULES, samplePercent: 0, droppedSignatures: [], priorities: { [GROUP]: 1 } } })
|
||||||
|
ix.add([post(1, 'Kitchen Cook <cook@example.com>', 'Soup.Recipes.2026.rar'), post(2, 'Kitchen Cook <cook@example.com>', 'Bread.Recipes.2026.rar'), post(3, 'Gardener <garden@example.com>', 'Roses.2026.rar'), post(4, 'Kitchen Helper <help@example.com>', 'Cake.2026.rar')], 5)
|
||||||
|
ix.finish()
|
||||||
|
await BatchStore.store(Number(range.id), Number(node.id), ix.take())
|
||||||
|
|
||||||
|
assert.sameMembers((await db.from('posters').select('poster')).map((p) => p.poster), ['Kitchen Cook <cook@example.com>', 'Gardener <garden@example.com>', 'Kitchen Helper <help@example.com>'])
|
||||||
|
const found = await SearchService.search({ poster: 'kitchen' })
|
||||||
|
assert.equal(found.total, 3)
|
||||||
|
assert.isTrue(found.releases.every((r) => r.poster?.startsWith('Kitchen')))
|
||||||
|
// Pages over the posters found, merged in the order asked.
|
||||||
|
const second = await SearchService.search({ poster: 'kitchen', perPage: 2, page: 2 })
|
||||||
|
assert.equal(second.total, 3)
|
||||||
|
assert.equal(second.releases.length, 1)
|
||||||
|
const all = [...(await SearchService.search({ poster: 'kitchen', perPage: 2, page: 1 })).releases, ...second.releases].map((r) => r.id)
|
||||||
|
assert.sameMembers(all, found.releases.map((r) => r.id))
|
||||||
|
assert.equal((await SearchService.search({ poster: 'cook', q: 'Bread' })).total, 1)
|
||||||
|
assert.equal((await SearchService.search({ poster: 'nobody' })).total, 0)
|
||||||
|
})
|
||||||
|
})
|
||||||
@@ -3,6 +3,7 @@ import db from '@adonisjs/lucid/services/db'
|
|||||||
import testUtils from '@adonisjs/core/services/test_utils'
|
import testUtils from '@adonisjs/core/services/test_utils'
|
||||||
import { ReleaseJudge } from '#services/release_judge'
|
import { ReleaseJudge } from '#services/release_judge'
|
||||||
import SearchService from '#services/search_service'
|
import SearchService from '#services/search_service'
|
||||||
|
import RangeBlocks from '#services/range_blocks'
|
||||||
|
|
||||||
test.group('ReleaseJudge', (group) => {
|
test.group('ReleaseJudge', (group) => {
|
||||||
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
group.each.setup(() => testUtils.db().withGlobalTransaction())
|
||||||
@@ -40,6 +41,9 @@ test.group('ReleaseJudge', (group) => {
|
|||||||
const fakeArchive = await release('Das.A-Team.S02E23.Das.Spiel.ist.aus')
|
const fakeArchive = await release('Das.A-Team.S02E23.Das.Spiel.ist.aus')
|
||||||
await inspect(fakeArchive, 'archive', { type: 'rar4', names: ['Das.A-Team.S02E23.Das.Spiel.ist.aus.exe'], entries: 1 })
|
await inspect(fakeArchive, 'archive', { type: 'rar4', names: ['Das.A-Team.S02E23.Das.Spiel.ist.aus.exe'], entries: 1 })
|
||||||
const fakeFile = await release('Some.Film.2021.1080p.WEB-DL')
|
const fakeFile = await release('Some.Film.2021.1080p.WEB-DL')
|
||||||
|
// Only "H.264" and "WEB-DL" say it's media here (a "?" in the pattern never matched).
|
||||||
|
const fakeCodec = await release('Some.Show.H.264.WEB-DL')
|
||||||
|
await inspect(fakeCodec, 'lone', { type: 'exe', yencName: 'setup.exe' })
|
||||||
await inspect(fakeFile, 'lone', { type: 'exe', yencName: 'Some.Film.2021.1080p.WEB-DL.mkv' })
|
await inspect(fakeFile, 'lone', { type: 'exe', yencName: 'Some.Film.2021.1080p.WEB-DL.mkv' })
|
||||||
// A program under a program's name is software, not fake.
|
// A program under a program's name is software, not fake.
|
||||||
const software = await release('LimeWire 4.19.6')
|
const software = await release('LimeWire 4.19.6')
|
||||||
@@ -53,13 +57,25 @@ test.group('ReleaseJudge', (group) => {
|
|||||||
await db.table('file_hashes').insert({ candidate_id: 20000 + i, release_id: id, hash16k: hash, size: 1073741824 })
|
await db.table('file_hashes').insert({ candidate_id: 20000 + i, release_id: id, hash16k: hash, size: 1073741824 })
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const scraped = await release('Jamie Lee Curtis Nude Celebrity Pictures', { poster: '@_CelebMovie_@', source: 'subject' })
|
||||||
|
const celeborn = await release('Some.Album.2010', { poster: 'celeborn@not.here (Celeborn)', source: 'subject' })
|
||||||
const locked = await release('Locked.Thing', { passworded: true })
|
const locked = await release('Locked.Thing', { passworded: true })
|
||||||
|
|
||||||
const unlocked = await release('Known.Password', { passworded: true })
|
const unlocked = await release('Known.Password', { passworded: true })
|
||||||
await db.table('release_passwords').insert({ release_id: unlocked, password: 'pw', source: 'spot-text' })
|
await db.table('release_passwords').insert({ release_id: unlocked, password: 'pw', source: 'spot-text' })
|
||||||
|
|
||||||
const unusable = await release('ab12CD34ef56GH78ij90', { readable: false })
|
// A block of done ranges around the unusable ones (release_final); one beside an unread range isn't judged yet.
|
||||||
const fresh = await release('zz12CD34ef56GH78ij90', { readable: false, settled_at: new Date() })
|
await db.table('groups').insert({ id: 1, name: 'alt.binaries.judged', priority: 1 }).onConflict('id').ignore()
|
||||||
const spotted = await release('yy12CD34ef56GH78ij90', { readable: false })
|
await db.table('ranges').multiInsert([
|
||||||
|
{ group_id: 1, start_id: 1, end_id: 1000, next_id: 1001, status: 'done', posted_low: new Date(settled.getTime() - 86400_000), posted_high: new Date(settled.getTime() + 86400_000) },
|
||||||
|
{ group_id: 1, start_id: 1001, end_id: 2000, next_id: 1001, status: 'planned' },
|
||||||
|
])
|
||||||
|
await RangeBlocks.rebuild()
|
||||||
|
const placed = { first_article: 100, last_article: 110 }
|
||||||
|
const unusable = await release('ab12CD34ef56GH78ij90', { readable: false, ...placed })
|
||||||
|
const pending = await release('pp12CD34ef56GH78ij90', { readable: false, first_article: 990, last_article: 1000, posted_at: new Date(settled.getTime() + 86400_000 - 3600_000) })
|
||||||
|
const fresh = await release('zz12CD34ef56GH78ij90', { readable: false, settled_at: new Date(), ...placed })
|
||||||
|
const spotted = await release('yy12CD34ef56GH78ij90', { readable: false, ...placed })
|
||||||
await db.table('release_spots').insert({ release_id: spotted, spot_id: 's@x', files: 1 })
|
await db.table('release_spots').insert({ release_id: spotted, spot_id: 's@x', files: 1 })
|
||||||
|
|
||||||
const round = await new ReleaseJudge().round()
|
const round = await new ReleaseJudge().round()
|
||||||
@@ -68,14 +84,19 @@ test.group('ReleaseJudge', (group) => {
|
|||||||
assert.isUndefined(verdicts.get(halfDead))
|
assert.isUndefined(verdicts.get(halfDead))
|
||||||
assert.equal(verdicts.get(fakeArchive), 'fake')
|
assert.equal(verdicts.get(fakeArchive), 'fake')
|
||||||
assert.equal(verdicts.get(fakeFile), 'fake')
|
assert.equal(verdicts.get(fakeFile), 'fake')
|
||||||
|
assert.equal(verdicts.get(fakeCodec), 'fake')
|
||||||
assert.isUndefined(verdicts.get(software))
|
assert.isUndefined(verdicts.get(software))
|
||||||
assert.deepEqual([...new Set(repeatedIds.map((id) => verdicts.get(id)))], ['repeated'])
|
assert.deepEqual([...new Set(repeatedIds.map((id) => verdicts.get(id)))], ['repeated'])
|
||||||
|
assert.equal(verdicts.get(scraped), 'scraped')
|
||||||
|
assert.isUndefined(verdicts.get(celeborn))
|
||||||
assert.equal(verdicts.get(locked), 'locked')
|
assert.equal(verdicts.get(locked), 'locked')
|
||||||
assert.isUndefined(verdicts.get(unlocked))
|
assert.isUndefined(verdicts.get(unlocked))
|
||||||
assert.equal(verdicts.get(unusable), 'unusable')
|
assert.equal(verdicts.get(unusable), 'unusable')
|
||||||
|
assert.isUndefined(verdicts.get(pending))
|
||||||
assert.isUndefined(verdicts.get(fresh))
|
assert.isUndefined(verdicts.get(fresh))
|
||||||
assert.isUndefined(verdicts.get(spotted))
|
assert.isUndefined(verdicts.get(spotted))
|
||||||
assert.equal(round.rules.repeated.added, 20)
|
assert.equal(round.rules.repeated.added, 20)
|
||||||
|
assert.equal(round.rules.scraped.matched, 1)
|
||||||
|
|
||||||
// Search leaves them out, unless asked.
|
// Search leaves them out, unless asked.
|
||||||
const found = await SearchService.search({ q: 'Locked' })
|
const found = await SearchService.search({ q: 'Locked' })
|
||||||
|
|||||||
@@ -222,6 +222,22 @@ test.group('release / par2 names', () => {
|
|||||||
assert.equal(mint('JimAndJoey Kindly Husband-to-Be.JPG').key, mint('Oops OnlyOneMore.JPG').key)
|
assert.equal(mint('JimAndJoey Kindly Husband-to-Be.JPG').key, mint('Oops OnlyOneMore.JPG').key)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
test("a folder's images posted with their sizes are one release: the subject names it", async ({ assert }) => {
|
||||||
|
const celeb = (n: string, size: number) =>
|
||||||
|
releaseIdentity({ poster: '@_CelebMovie_@', subject: `>>>>> Jamie Lee Curtis - Nude Celebrity Pictures <<<<< - ${n} ${size} bytes`, name: n, weak: false })!
|
||||||
|
const keys = new Set(['!index0001.jpg', '00-00-02_123_842lo.jpg', '00_00_13.jpg', 'bboobs02-curtis-jl.jpg', '0c4_jamie_lee_curtis_15.jpg'].map((n, i) => celeb(n, 75220 + i).key))
|
||||||
|
assert.equal(keys.size, 1)
|
||||||
|
assert.equal(celeb('00_00_13.jpg', 58748).source, 'subject')
|
||||||
|
assert.notEqual(celeb('00_00_13.jpg', 1).key, releaseIdentity({ poster: '@_CelebMovie_@', subject: '>>>>> Carmen Electra - Xmas Nudes <<<<< - 00_00_13.jpg 1 bytes', name: '00_00_13.jpg', weak: false })!.key)
|
||||||
|
const ctp = (n: string) => releaseIdentity({ poster: 'new@update.org (New CTP update)', subject: `*** NEW *** CTP update & adds - ${n} - 18441 bytes`, name: n, weak: false })!
|
||||||
|
assert.equal(ctp('cc_CG2008_v10_01_Stephanie_Emma.jpg').key, ctp('cc_CastingCall_LasVegas_01_Kristin_00_Header1.jpg').key)
|
||||||
|
const { sizedName } = await import('#lib/release/identity')
|
||||||
|
assert.isTrue(sizedName('x - a.jpg 1,234 Bytes', 'a.jpg'))
|
||||||
|
assert.isFalse(sizedName('x - "a.jpg" yEnc 320794 bytes', 'a.jpg'))
|
||||||
|
assert.isFalse(sizedName('x - a.jpg 5 megabytes', 'a.jpg'))
|
||||||
|
assert.isFalse(sizedName('x - a.jpg yEnc', 'a.jpg'))
|
||||||
|
})
|
||||||
|
|
||||||
test("an image's set: suffixes that don't count the images go first", async ({ assert }) => {
|
test("an image's set: suffixes that don't count the images go first", async ({ assert }) => {
|
||||||
const { imageSet } = await import('#lib/release/identity')
|
const { imageSet } = await import('#lib/release/identity')
|
||||||
for (const [stem, set] of [
|
for (const [stem, set] of [
|
||||||
|
|||||||
@@ -3,7 +3,13 @@ import { DEFAULT_RULES, RuleSet, ownerOf, uploadTool, explored } from '#lib/rule
|
|||||||
import { sevenZipVolume } from '#lib/files/index'
|
import { sevenZipVolume } from '#lib/files/index'
|
||||||
|
|
||||||
const rules = new RuleSet({ ...DEFAULT_RULES, samplePercent: 0, priorities: {}, droppedSignatures: ['nyuu\talt.binaries.misc'] })
|
const rules = new RuleSet({ ...DEFAULT_RULES, samplePercent: 0, priorities: {}, droppedSignatures: ['nyuu\talt.binaries.misc'] })
|
||||||
const file = (name: string, totalParts: number, firstMessageId: string, group = 'alt.binaries.ath') => ({ name, totalParts, groups: [group], firstMessageId })
|
const file = (name: string, totalParts: number, firstMessageId: string, group = 'alt.binaries.ath', poster = 'someone@example.com (Someone)') => ({
|
||||||
|
name,
|
||||||
|
totalParts,
|
||||||
|
groups: [group],
|
||||||
|
firstMessageId,
|
||||||
|
poster,
|
||||||
|
})
|
||||||
const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost'
|
const NGPOST = 'b94268bc702040f7b811d5318f727e4f@ngPost'
|
||||||
const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'
|
const NYUU = 'FmFwIaTpGgUiSfYcMwFlMeRo-1790637260478@nyuu'
|
||||||
|
|
||||||
@@ -17,6 +23,17 @@ test.group('rules / owner', () => {
|
|||||||
})
|
})
|
||||||
|
|
||||||
test.group('rules / verdicts', () => {
|
test.group('rules / verdicts', () => {
|
||||||
|
test("a dropped poster's uploads are dropped whatever they are; others with the word in their name are not", ({ assert }) => {
|
||||||
|
const pics = (poster: string) => rules.judge('k', [file('!index0001.jpg', 1, NYUU, 'alt.binaries.erotica', poster)])
|
||||||
|
assert.include(pics('@_CelebMovie_@'), { rule: 'poster', key: 'celebmovie' })
|
||||||
|
assert.include(pics('@__Celeb_Movie__@'), { rule: 'poster', key: 'celebmovie' })
|
||||||
|
assert.include(pics('NudeCelebs@CelebWow.com'), { rule: 'poster', key: 'celebwow' })
|
||||||
|
assert.include(pics('@nude-celebs'), { rule: 'poster', key: 'nudecelebs' })
|
||||||
|
assert.deepEqual(pics('celeborn@not.here (Celeborn)'), { rule: null })
|
||||||
|
assert.deepEqual(pics('"CelebTA.com" <from@celebta.com>'), { rule: null })
|
||||||
|
assert.deepEqual(new RuleSet({ ...DEFAULT_RULES, droppedPosters: [], samplePercent: 0, priorities: {}, droppedSignatures: [] }).judge('k', [file('a.jpg', 1, NYUU, 'g', '@_CelebMovie_@')]), { rule: null })
|
||||||
|
})
|
||||||
|
|
||||||
test('a readable file keeps its whole release', ({ assert }) => {
|
test('a readable file keeps its whole release', ({ assert }) => {
|
||||||
assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST), file('Some.Show.S01E01.1080p.nfo', 1, NGPOST)]), { rule: null })
|
assert.deepEqual(rules.judge('k', [file('Hs7dKq2LmZ9xWv3RtYp1Qa.part01.rar', 50, NGPOST), file('Some.Show.S01E01.1080p.nfo', 1, NGPOST)]), { rule: null })
|
||||||
})
|
})
|
||||||
|
|||||||
Reference in new issue
Block a user