diff --git a/REBUILD.md b/REBUILD.md index 259cfa4..d58c1e5 100644 --- a/REBUILD.md +++ b/REBUILD.md @@ -247,9 +247,9 @@ with 68,032 releases with a password): | Feature | Voyager 1 | In voyager 2 | |---|---|---| -| Spotnet spots | D29 | not yet; 2.76 M spots in the export | +| Spotnet spots | D29 | the spotter and the spot inspector (below); new spots only, the export's not imported | | Posted NZBs read into releases (also NZBs inside archives) | D26, D28 | not yet | -| Passwords (from spots, NZB meta, names) | D29, D26 | only *whether* passworded (the prober); the export's 68 k passwords not imported | +| Passwords (from spots, NZB meta, names) | D29, D26 | from spots (title, text) and their NZBs' meta, in `release_passwords`; else only *whether* passworded (the prober); the export's 68 k not imported | | SFV grouping and naming | D34 | not yet (SFV files aren't probe candidates) | | Complete RAR listings by walking volume headers | D40 | no: the prober reads the first volume's first article only | | 7z listings | D19 | no: `7z-tail` candidates are queued, not probed | @@ -258,6 +258,37 @@ with 68,032 releases with a password): Live indexing (the tail) runs since 2026-10-09. +**Spots** (after voyager 1's D29; `lib/spotnet` ported as is). Two +processes, one NNTP connection each: + +- **The spotter** (`spots:read`, service `spotter`) reads free.pt's + overviews after the last article it saw (`spot_state`; the first time the + last `SPOT_BACKFILL`, 20,000, ~3 months), every `SPOT_EVERY_SECONDS` + (120). For each spot it reads the full header: the signature is checked + there as Spotweb does (Spotnet's keys 2–4; self-signed key 7 by hashcash), + and the X-XML gives the description, the website and the NZB's articles. + A password the title (`{{pw}}`) or the description names + (`passwordFromSpot`) is kept with the spot, with its source. A + delete/dispose message marks its target `removed` (a personal one only + from the same spotter within 5 days) and takes its links and passwords + back from the releases. +- **The spot inspector** (`spots:inspect`, service `spot-inspector`) looks + inside each listed spot, newest first: it reads the compressed NZB (≤ + `SPOT_MAX_NZB_SEGMENTS` articles in alt.binaries.ftd), lists its files + in `spot_files` (named as the header pass names them, `parseSubject`), + and matches each file to the release the header pass built for it: the + same release identity (`releaseIdentity` over the file's subject and + poster) in one of its groups, the release posted nearest the file within + 48 h. Matched releases are linked in `release_spots`. Placeholder NZBs + (`dummy`), gone articles (`nzb-gone`) and unreadable ones are noted on the + spot. Unmatched spots are matched again every hour; after + `SPOT_STANDALONE_AFTER_HOURS` (24) a spot becomes a release of its own + (source `spot`), whose NZB is the spot's (kept gzipped in `spots.nzb`). +- **Passwords are kept apart** in `release_passwords` (release, password, + source: `nzb` for the NZB's ``, else `spot-title` or + `spot-text`, and the spot), not on the catalog row; the catalog only says + `passworded`. + ## Decisions a. Chunk payload: a write-once `bytea` table in Postgres, partitioned by diff --git a/app/services/nzb_service.ts b/app/services/nzb_service.ts index 2f82dcb..90c1b19 100644 --- a/app/services/nzb_service.ts +++ b/app/services/nzb_service.ts @@ -1,3 +1,4 @@ +import { gunzipSync } from 'node:zlib' import db from '@adonisjs/lucid/services/db' import { decodeChunk, type ChunkFile } from '#lib/chunks/index' import { buildNzb } from '#lib/nzb/index' @@ -19,8 +20,13 @@ class NzbService { } async build(releaseId: number): Promise<{ name: string; xml: string } | null> { - const release = await db.from('releases').where('id', releaseId).select('name').first() + const release = await db.from('releases').where('id', releaseId).select('name', 'source').first() if (!release) return null + // A release a spot made of its own (no files matched the header pass's): the spot's NZB as read. + if (release.source === 'spot') { + const spot = await db.from('spots').where('release_id', releaseId).whereNotNull('nzb').select('nzb').first() + if (spot) return { name: release.name, xml: gunzipSync(spot.nzb).toString('utf8') } + } const xml = buildNzb( (await this.files(releaseId)).map((f) => ({ name: f.name, diff --git a/app/services/spot_inspector.ts b/app/services/spot_inspector.ts new file mode 100644 index 0000000..4ee9471 --- /dev/null +++ b/app/services/spot_inspector.ts @@ -0,0 +1,331 @@ +import { createHash } from 'node:crypto' +import { gzipSync } from 'node:zlib' +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import env from '#start/env' +import coordinatorConfig from '#config/coordinator' +import { XoverClient } from '#lib/nntp/index' +import { decodeSpotBinary } from '#lib/spotnet/index' +import { parseNzb, type ParsedNzb } from '#lib/nzb/index' +import { parseSubject } from '#lib/headers/index' +import { releaseIdentity } from '#lib/release/index' + +/** Spots claimed per round, newest first. */ +const BATCH = 20 +/** Between rounds when there was nothing to do. */ +const IDLE_MS = 60_000 +/** Unmatched spots are matched again this often (the header pass may have built their releases since). */ +const REMATCH_MS = 60 * 60 * 1000 +/** A file's release posted further apart than this from the NZB's date is another upload of the same name. */ +const MAX_SPREAD_MS = 48 * 3600 * 1000 + +export interface InspectRound { + inspected: number + matched: number + standalone: number + outcomes: Record +} + +interface SpotFile { + idx: number + name: string + subject: string + poster: string + groups: string[] + postedAt: Date | null + bytes: number + segments: number +} + +/** + * A placeholder NZB (Dummy.rar + Dummy.nzb, shared by many spots): under 1% of the size the spot announces, or under + * 16 KB in all (text-only spots). Its files aren't listed (voyager 1's isDummyNzb). + */ +export function isDummyNzb(nzb: ParsedNzb, spotSize: number): boolean { + const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0) + return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100) +} + +/** A file of an NZB as the header pass names it: the subject's file name, by the same parser. */ +export function nzbFileName(subject: string): { name: string; base: string } { + const parsed = parseSubject(subject) ?? parseSubject(`${subject} (1/1)`) + return parsed ? { name: parsed.name, base: parsed.base } : { name: subject, base: subject } +} + +/** + * The spot inspector (REBUILD.md, spots): looks inside each listed spot, on + * one NNTP connection of its own. It reads the spot's compressed NZB (its + * articles in alt.binaries.ftd), keeps the list of files in `spot_files`, + * and matches every file to the release the header pass built for it: the + * release identity of the file's subject and poster (lib/release, as the + * edge workers compute it), in one of the file's groups, posted within 48 h. + * Each matched release is linked in `release_spots`, and gets the spot's + * password in `release_passwords` (the NZB's own `` + * first, else what the spot's title or text names) and `passworded`. A spot + * whose files match nothing is matched again every hour; after + * SPOT_STANDALONE_AFTER_HOURS it becomes a release of its own (source `spot`) + * with its NZB. + */ +export class SpotInspector { + private stopped = false + private client?: XoverClient + private groupIds = new Map() + private rematchedAt = 0 + + stop() { + this.stopped = true + this.client?.close() + } + + async run() { + while (!this.stopped) { + let round: InspectRound | null = null + try { + round = await this.round() + if (round.inspected > 0 || round.standalone > 0) { + const outcomes = Object.entries(round.outcomes).map(([k, n]) => `${n} ${k}`).join(', ') + logger.info(`Spots inspected: ${round.inspected} (${outcomes}); ${round.matched} matched releases, ${round.standalone} of their own.`) + } + } catch (error: any) { + logger.error(`Spot inspection failed: ${error.message}`) + this.client?.close() + this.client = undefined + } + if (!round || round.inspected === 0) { + for (let waited = 0; waited < IDLE_MS && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000)) + } + } + } + + private async connect(): Promise { + this.client ??= await XoverClient.connect({ + host: env.get('NNTP_HOST')!, + port: env.get('NNTP_PORT', 563), + user: env.get('NNTP_USER'), + password: env.get('NNTP_PASSWORD'), + secure: env.get('NNTP_SECURE', true), + compress: false, + timeoutMs: coordinatorConfig.nntpTimeoutMs, + }) + return this.client + } + + async round(): Promise { + const round: InspectRound = { inspected: 0, matched: 0, standalone: 0, outcomes: {} } + const spots = await db.from('spots').where('status', 'listed').orderBy('posted_at', 'desc').limit(BATCH) + for (const spot of spots) { + if (this.stopped) break + const outcome = await this.inspect(spot) + round.inspected++ + round.outcomes[outcome.status] = (round.outcomes[outcome.status] ?? 0) + 1 + round.matched += outcome.matched + } + if (Date.now() - this.rematchedAt >= REMATCH_MS && !this.stopped) { + this.rematchedAt = Date.now() + const again = await this.rematch() + round.matched += again.matched + round.standalone += again.standalone + } + return round + } + + /** One spot: its NZB read, its files listed and matched. */ + async inspect(spot: any): Promise<{ status: string; matched: number }> { + const done = async (status: string, extra: Record = {}) => { + await db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra }) + return { status, matched: 0 } + } + const segments: string[] = spot.nzb_segments ?? [] + if (segments.length === 0) return done('no-nzb') + if (segments.length > coordinatorConfig.spots.maxNzbSegments) return done('no-nzb', { error: `${segments.length} NZB articles` }) + const client = await this.connect() + const bodies = await client.bodies(segments) + if (bodies.some((b) => b === null)) return done('nzb-gone', { error: `${bodies.filter((b) => b === null).length} of ${segments.length} NZB articles gone` }) + let xml: string + try { + xml = decodeSpotBinary(bodies as Buffer[]).toString('utf8') + } catch (error: any) { + return done('nzb-unreadable', { error: String(error.message).slice(0, 500) }) + } + return this.inspectNzb(spot, xml) + } + + /** The files of a spot's NZB listed and matched (the tests feed this directly). */ + async inspectNzb(spot: any, xml: string): Promise<{ status: string; matched: number }> { + const nzb = parseNzb(xml) + const finish = (status: string, extra: Record = {}) => + db.from('spots').where('message_id', spot.message_id).update({ status, inspected_at: new Date(), ...extra }) + if (!nzb || nzb.files.length === 0) { + await finish('no-nzb') + return { status: 'no-nzb', matched: 0 } + } + if (isDummyNzb(nzb, Number(spot.size ?? 0))) { + await finish('dummy') + return { status: 'dummy', matched: 0 } + } + const files: SpotFile[] = nzb.files.map((f, idx) => ({ + idx, + name: nzbFileName(f.subject).name.slice(0, 1000), + subject: f.subject.slice(0, 2000), + poster: f.poster, + groups: f.groups, + postedAt: f.date ? new Date(f.date * 1000) : null, + bytes: f.segments.reduce((s, x) => s + x.bytes, 0), + segments: f.segments.length, + })) + // The NZB's own password over what the spot's text names. + const password = nzb.meta.password ? { password: nzb.meta.password.slice(0, 200), source: 'nzb' } : spot.password ? { password: spot.password, source: spot.password_source } : null + await db.transaction(async (trx) => { + await trx.from('spot_files').where('spot_id', spot.message_id).delete() + for (let i = 0; i < files.length; i += 1000) { + await trx.table('spot_files').multiInsert( + files.slice(i, i + 1000).map((f) => ({ + spot_id: spot.message_id, + idx: f.idx, + name: f.name, + subject: f.subject, + poster: f.poster, + groups: f.groups, + posted_at: f.postedAt, + bytes: f.bytes, + segments: f.segments, + })) + ) + } + await trx + .from('spots') + .where('message_id', spot.message_id) + .update({ + status: 'inspected', + inspected_at: new Date(), + files: files.length, + nzb: gzipSync(xml), + ...(password && password.source === 'nzb' ? { password: password.password, password_source: 'nzb' } : {}), + }) + }) + const matched = await this.match(spot.message_id) + return { status: 'inspected', matched } + } + + private async groupId(name: string): Promise { + if (!this.groupIds.has(name)) { + const row = await db.from('groups').where('name', name).select('id').first() + this.groupIds.set(name, row ? Number(row.id) : null) + } + return this.groupIds.get(name)! + } + + /** + * Matches a spot's listed files to releases and links them; how many releases. A file's release: the same + * identity (releaseIdentity over its subject and poster) in one of its groups, the one posted nearest the file. + */ + async match(spotId: string): Promise { + const spot = await db.from('spots').where('message_id', spotId).first() + if (!spot || spot.status !== 'inspected') return 0 + const files = await db.from('spot_files').where('spot_id', spotId).orderBy('idx') + const wanted: { idx: number; key: string; groups: number[]; at: number | null }[] = [] + for (const f of files) { + if (!f.subject) continue + const { name, base } = nzbFileName(f.subject) + const identity = releaseIdentity({ poster: f.poster ?? '', subject: base, name, weak: false }) + if (!identity) continue + const groups = (await Promise.all((f.groups ?? []).map((g: string) => this.groupId(g)))).filter((g): g is number => g !== null) + if (groups.length > 0) wanted.push({ idx: f.idx, key: identity.key, groups, at: f.posted_at ? new Date(f.posted_at).getTime() : null }) + } + if (wanted.length === 0) return 0 + const { rows } = await db.rawQuery( + `SELECT id, identity_key, group_id, posted_at FROM releases WHERE identity_key = ANY(?) AND group_id = ANY(?)`, + [[...new Set(wanted.map((w) => w.key))], [...new Set(wanted.flatMap((w) => w.groups))]] + ) + const byKey = new Map() + for (const r of rows) { + const list = byKey.get(r.identity_key) ?? [] + list.push({ id: Number(r.id), group: Number(r.group_id), at: r.posted_at ? new Date(r.posted_at).getTime() : null }) + byKey.set(r.identity_key, list) + } + const fileRelease = new Map() + for (const w of wanted) { + const near = (byKey.get(w.key) ?? []) + .filter((r) => w.groups.includes(r.group)) + .map((r) => ({ ...r, spread: r.at === null || w.at === null ? 0 : Math.abs(r.at - w.at) })) + .filter((r) => r.spread <= MAX_SPREAD_MS) + .sort((a, b) => a.spread - b.spread) + if (near[0]) fileRelease.set(w.idx, near[0].id) + } + const perRelease = new Map() + for (const id of fileRelease.values()) perRelease.set(id, (perRelease.get(id) ?? 0) + 1) + await db.transaction(async (trx) => { + for (const [idx, releaseId] of fileRelease) await trx.from('spot_files').where({ spot_id: spotId, idx }).update({ release_id: releaseId }) + for (const [releaseId, n] of perRelease) { + await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: n }).onConflict(['release_id', 'spot_id']).merge({ files: n }) + } + if (spot.password && perRelease.size > 0) { + await trx + .table('release_passwords') + .multiInsert([...perRelease.keys()].map((id) => ({ release_id: id, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }))) + .onConflict(['release_id', 'password']) + .ignore() + await trx.from('releases').whereIn('id', [...perRelease.keys()]).update({ passworded: true }) + } + const top = [...perRelease.entries()].sort((a, b) => b[1] - a[1])[0] + await trx.from('spots').where('message_id', spotId).update({ matched: fileRelease.size, release_id: top ? top[0] : null }) + }) + return perRelease.size + } + + /** Unmatched inspected spots matched again; the ones that waited long enough become releases of their own. */ + async rematch(): Promise<{ matched: number; standalone: number }> { + const out = { matched: 0, standalone: 0 } + const spots = await db.from('spots').where('status', 'inspected').where('matched', 0).whereNull('release_id').orderBy('inspected_at').limit(500) + const cutoff = Date.now() - coordinatorConfig.spots.standaloneAfterHours * 3600 * 1000 + for (const spot of spots) { + if (this.stopped) break + const n = await this.match(spot.message_id) + out.matched += n + if (n === 0 && spot.posted_at && new Date(spot.posted_at).getTime() < cutoff && (await this.standalone(spot.message_id))) out.standalone++ + } + return out + } + + /** A release of the spot's own (source `spot`), from its listed files; false when it has none. */ + async standalone(spotId: string): Promise { + const spot = await db.from('spots').where('message_id', spotId).first() + const files = await db.from('spot_files').where('spot_id', spotId) + if (!spot || files.length === 0) return false + const groups = [...new Set(files.flatMap((f: any) => f.groups ?? []))] + const groupId = groups.length > 0 ? await this.groupId(groups[0]) : null + const key = createHash('sha1').update(`spot\n${spotId}`).digest('hex') + const [row] = await db + .table('releases') + .insert({ + release_key: `spot:${key}`, + identity_key: key, + group_id: groupId, + name: spot.title.slice(0, 1000), + source: 'spot', + poster: spot.poster, + groups: JSON.stringify(groups), + posted_at: spot.posted_at, + last_posted_at: spot.posted_at, + files: files.length, + segments: files.reduce((s: number, f: any) => s + Number(f.segments), 0), + bytes: files.reduce((s: number, f: any) => s + Number(f.bytes), 0), + readable: true, + passworded: spot.password ? true : null, + settled_at: new Date(), + }) + .onConflict('release_key') + .ignore() + .returning('id') + if (!row) return false + const releaseId = Number(row.id) + await db.transaction(async (trx) => { + await trx.table('release_spots').insert({ release_id: releaseId, spot_id: spotId, files: files.length }).onConflict(['release_id', 'spot_id']).ignore() + if (spot.password) { + await trx.table('release_passwords').insert({ release_id: releaseId, password: spot.password, source: spot.password_source ?? 'spot-text', spot_id: spotId }).onConflict(['release_id', 'password']).ignore() + } + await trx.from('spots').where('message_id', spotId).update({ release_id: releaseId }) + }) + return true + } +} diff --git a/app/services/spotter.ts b/app/services/spotter.ts new file mode 100644 index 0000000..0520a36 --- /dev/null +++ b/app/services/spotter.ts @@ -0,0 +1,184 @@ +import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' +import env from '#start/env' +import coordinatorConfig from '#config/coordinator' +import { MissingArticle, XoverClient } from '#lib/nntp/index' +import { + decodeSpotText, + moderationTarget, + parseSpotArticleHeaders, + parseSpotHeader, + parseSpotXml, + passwordFromSpot, + spotterId, + verifySpotHeader, +} from '#lib/spotnet/index' + +/** Overviews asked for at a time. */ +const XOVER_STEP = 1000 +/** A personal dispose counts within this long of the spot (Spotweb). */ +const DISPOSE_WINDOW_SECONDS = 5 * 86400 + +export interface SpotterRound { + read: number + listed: number + unverified: number + moderation: number +} + +/** + * The spotter (REBUILD.md, spots): Spotnet's spots, from the headers of + * free.pt into `spots`, on one NNTP connection of its own. Every few + * minutes it reads the overviews after the last article it saw (the first + * time the last SPOT_BACKFILL), and for each spot the full header: its + * signature is checked there (lib/spotnet, as Spotweb does), and the X-XML + * gives the description, the website and the NZB's articles. A password the + * title (`{{pw}}`) or the description names is kept with the spot; the spot + * inspector carries it to the releases. A delete/dispose message marks its + * target `removed` and takes back what the target gave the releases. + */ +export class Spotter { + private stopped = false + private client?: XoverClient + + stop() { + this.stopped = true + this.client?.close() + } + + async run() { + while (!this.stopped) { + const started = Date.now() + try { + const round = await this.round() + if (round.read > 0) { + logger.info(`Spots: ${round.read} articles read, ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`) + } + } catch (error: any) { + logger.error(`Spot round failed: ${error.message}`) + this.client?.close() + this.client = undefined + } + const wait = coordinatorConfig.spots.everySeconds * 1000 + for (let waited = Date.now() - started; waited < wait && !this.stopped; waited += 1000) await new Promise((r) => setTimeout(r, 1000)) + } + } + + private async connect(): Promise { + this.client ??= await XoverClient.connect({ + host: env.get('NNTP_HOST')!, + port: env.get('NNTP_PORT', 563), + user: env.get('NNTP_USER'), + password: env.get('NNTP_PASSWORD'), + secure: env.get('NNTP_SECURE', true), + compress: false, + timeoutMs: coordinatorConfig.nntpTimeoutMs, + }) + return this.client + } + + /** The articles after the last one seen, up to the group's high. */ + async round(): Promise { + const { group, backfill } = coordinatorConfig.spots + const client = await this.connect() + const { low, high } = await client.group(group) + const state = await db.from('spot_state').where('group_name', group).first() + let next = state ? Number(state.last_article) + 1 : Math.max(low, high - backfill + 1) + const round: SpotterRound = { read: 0, listed: 0, unverified: 0, moderation: 0 } + while (next <= high && !this.stopped) { + const to = Math.min(high, next + XOVER_STEP - 1) + const overviews = await client.xover(next, to) + round.read += overviews.length + for (const [number, fields] of overviews) { + if (this.stopped) return round + const messageId = (fields['message-id'] ?? '').replace(/^<|>$/g, '') + if (!messageId || !parseSpotHeader(fields.from ?? '', fields.subject ?? '', messageId, fields.date)) continue + const outcome = await this.read(client, messageId, number) + if (outcome) round[outcome]++ + } + await db + .table('spot_state') + .insert({ group_name: group, last_article: to }) + .onConflict('group_name') + .merge({ last_article: to }) + next = to + 1 + } + return round + } + + /** One spot by its full header; what it was, or null when it was there already or is gone. */ + async read(client: XoverClient, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> { + if (await db.from('spots').where('message_id', messageId).first()) return null + let raw: Buffer + try { + raw = await client.head(messageId) + } catch (error) { + if (error instanceof MissingArticle) return null + throw error + } + return this.store(raw.toString('latin1'), messageId, articleNumber) + } + + /** Verifies and stores a spot from its raw header (the tests feed this directly). */ + async store(raw: string, messageId: string, articleNumber: number | null): Promise<'listed' | 'unverified' | 'moderation' | null> { + const { headers, xml: xmlText } = parseSpotArticleHeaders(raw) + const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date) + if (!header) return null + const verified = verifySpotHeader(header) + const xml = verified ? parseSpotXml(decodeSpotText(xmlText)) : null + const target = verified ? moderationTarget(header) : null + const title = xml?.title || header.title + // Kept apart by where it came from: the title's {{pw}}, else the description's "Wachtwoord: …". + const fromTitle = verified && !target ? passwordFromSpot(title, null) : null + const fromText = verified && !target && !fromTitle ? passwordFromSpot('', xml?.description) : null + const status = !verified ? 'unverified' : target ? 'moderation' : 'listed' + await db + .table('spots') + .insert({ + message_id: messageId, + article_number: articleNumber, + title: title.slice(0, 2000), + tag: header.tag || null, + poster: header.poster, + category: header.category, + subcats: JSON.stringify(header.subcats), + size: header.size || null, + posted_at: header.stamp ? new Date(header.stamp * 1000) : null, + key_id: header.keyId, + spotter_id: header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : null, + description: xml?.description || null, + website: xml?.website || null, + nzb_segments: xml?.nzbSegments.length ? xml.nzbSegments : null, + password: fromTitle ?? fromText, + password_source: fromTitle ? 'spot-title' : fromText ? 'spot-text' : null, + status, + }) + .onConflict('message_id') + .ignore() + if (target) await this.moderate(header.selfSignedPubKey, header.stamp, target) + return status + } + + /** + * A delete/dispose/remove: the target is `removed`, and its links and passwords go from the releases. As in + * Spotweb, a personal dispose (the poster's own key) counts only from the same spotter within 5 days; a + * moderator's (Spotnet's key) always. + */ + private async moderate(pubKey: string, stamp: number, targetId: string) { + const target = await db.from('spots').where('message_id', targetId).first() + if (!target) return + const personal = pubKey.length > 50 ? spotterId(pubKey) : '' + if (personal) { + const age = stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0) + if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return + } + await db.transaction(async (trx) => { + await trx.from('spots').where('message_id', targetId).update({ status: 'removed' }) + await trx.from('release_spots').where('spot_id', targetId).delete() + await trx.from('release_passwords').where('spot_id', targetId).delete() + // A release the spot made of its own goes with it. + if (target.release_id) await trx.from('releases').where('id', target.release_id).where('source', 'spot').delete() + }) + logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`) + } +} diff --git a/commands/spots_inspect.ts b/commands/spots_inspect.ts new file mode 100644 index 0000000..921f0b1 --- /dev/null +++ b/commands/spots_inspect.ts @@ -0,0 +1,33 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import { SpotInspector } from '#services/spot_inspector' + +/** The spot inspector (REBUILD.md, spots): each spot's NZB read, its files listed and matched to releases. */ +export default class SpotsInspect extends BaseCommand { + static commandName = 'spots:inspect' + static description = "Reads each listed spot's NZB, lists its files and links them to the releases they match" + static options: CommandOptions = { startApp: true, staysAlive: true } + + @flags.boolean({ description: 'One round, then exit' }) + declare once: boolean + + async run() { + const inspector = new SpotInspector() + if (this.once) { + const round = await inspector.round() + this.logger.info(`${round.inspected} spots inspected, ${round.matched} matched releases, ${round.standalone} of their own.`) + inspector.stop() + await this.terminate() + return + } + let done: () => void + const finished = new Promise((resolve) => (done = resolve)) + this.app.terminating(async () => { + inspector.stop() + await finished + }) + await inspector.run() + this.logger.info('Spot inspector stopped.') + done!() + } +} diff --git a/commands/spots_read.ts b/commands/spots_read.ts new file mode 100644 index 0000000..d26d754 --- /dev/null +++ b/commands/spots_read.ts @@ -0,0 +1,33 @@ +import { BaseCommand, flags } from '@adonisjs/core/ace' +import type { CommandOptions } from '@adonisjs/core/types/ace' +import { Spotter } from '#services/spotter' + +/** The spotter (REBUILD.md, spots): Spotnet's spots from free.pt's headers into `spots`, on one connection. */ +export default class SpotsRead extends BaseCommand { + static commandName = 'spots:read' + static description = "Reads Spotnet's spots (free.pt) into the spots table, verified, every few minutes" + static options: CommandOptions = { startApp: true, staysAlive: true } + + @flags.boolean({ description: 'One round, then exit' }) + declare once: boolean + + async run() { + const spotter = new Spotter() + if (this.once) { + const round = await spotter.round() + this.logger.info(`${round.read} articles read: ${round.listed} spots listed, ${round.unverified} unverified, ${round.moderation} moderation.`) + spotter.stop() + await this.terminate() + return + } + let done: () => void + const finished = new Promise((resolve) => (done = resolve)) + this.app.terminating(async () => { + spotter.stop() + await finished + }) + await spotter.run() + this.logger.info('Spotter stopped.') + done!() + } +} diff --git a/config/coordinator.ts b/config/coordinator.ts index df81f60..3e354dc 100644 --- a/config/coordinator.ts +++ b/config/coordinator.ts @@ -77,6 +77,19 @@ export default { /** BODY commands in flight per connection (pipelined; XoverClient.bodies). */ pipeline: env.get('PROBE_PIPELINE', 8), }, + /** + * Spotnet spots (app/services/spotter.ts, spot_inspector.ts): the group the spots are posted in, how many of + * its last articles the spotter reads the first time (~215 spots a day: ~3 months), how often it looks again, + * the most articles an NZB may take, and how long an inspected spot whose files match no release waits for + * the header pass before it becomes a release of its own. + */ + spots: { + group: env.get('SPOT_GROUP', 'free.pt'), + backfill: env.get('SPOT_BACKFILL', 20_000), + everySeconds: env.get('SPOT_EVERY_SECONDS', 120), + maxNzbSegments: env.get('SPOT_MAX_NZB_SEGMENTS', 64), + standaloneAfterHours: env.get('SPOT_STANDALONE_AFTER_HOURS', 24), + }, /** Stored XOVER chunks of ranges marked `capture`, for the grouping sweeps (app/services/corpus.ts). Off when unset. */ corpus: { dir: env.get('CORPUS_DIR'), diff --git a/database/migrations/1770000002800_spots.ts b/database/migrations/1770000002800_spots.ts new file mode 100644 index 0000000..df681c1 --- /dev/null +++ b/database/migrations/1770000002800_spots.ts @@ -0,0 +1,95 @@ +import { BaseSchema } from '@adonisjs/lucid/schema' + +/** + * Spotnet spots (REBUILD.md, spots; after voyager 1's D29), in two processes: + * - the spotter (`spots:read`) reads free.pt's headers, verifies each spot's + * signature from its full header (X-XML) and keeps it in `spots`, with a + * password its title or description names; + * - the spot inspector (`spots:inspect`) reads each spot's compressed NZB + * (alt.binaries.ftd), keeps its files in `spot_files`, and matches them to + * the releases the header pass built (the same release identity), linking + * them in `release_spots`. A spot whose files match no release becomes a + * release of its own (source `spot`) once a day has passed. + * The NZB itself is kept (gzip) for such a release's downloads. + * Passwords are kept apart from the catalog in `release_passwords`, one row + * per release and password, with where it came from (the NZB's meta, the + * spot's title or its text); the catalog only says `passworded`. + */ +export default class extends BaseSchema { + async up() { + this.schema.raw(` + CREATE TABLE spots ( + message_id text PRIMARY KEY, + article_number bigint, + title text NOT NULL, + tag text, + poster text NOT NULL, + category integer, + subcats jsonb, + size bigint, + posted_at timestamptz, + key_id integer, + spotter_id text, + description text, + website text, + nzb_segments text[], + -- The NZB as read (gzip): a release the spot makes of its own has no chunks to build one from. + nzb bytea, + password text, + password_source varchar(16), + -- listed: verified, waiting for the inspector; inspected, no-nzb, dummy, nzb-gone, nzb-unreadable: after it; + -- moderation: a delete/dispose message; removed: disposed of by a moderator or its spotter. + status varchar(16) NOT NULL DEFAULT 'listed', + files integer, + matched integer, + release_id bigint, + error text, + created_at timestamptz NOT NULL DEFAULT now(), + inspected_at timestamptz + )`) + this.schema.raw(`CREATE INDEX spots_listed_idx ON spots (posted_at DESC) WHERE status = 'listed'`) + this.schema.raw(`CREATE INDEX spots_unmatched_idx ON spots (inspected_at) WHERE status = 'inspected' AND matched = 0 AND release_id IS NULL`) + this.schema.raw(` + CREATE TABLE spot_files ( + spot_id text NOT NULL, + idx integer NOT NULL, + name text NOT NULL, + subject text NOT NULL, + poster text, + groups text[], + posted_at timestamptz, + bytes bigint NOT NULL, + segments integer NOT NULL, + release_id bigint, + PRIMARY KEY (spot_id, idx) + )`) + this.schema.raw(`CREATE INDEX spot_files_release_idx ON spot_files (release_id) WHERE release_id IS NOT NULL`) + this.schema.raw(` + CREATE TABLE release_spots ( + release_id bigint NOT NULL, + spot_id text NOT NULL, + files integer NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (release_id, spot_id) + )`) + this.schema.raw(`CREATE INDEX release_spots_spot_idx ON release_spots (spot_id)`) + this.schema.raw(` + CREATE TABLE release_passwords ( + release_id bigint NOT NULL, + password text NOT NULL, + source varchar(16) NOT NULL, + spot_id text, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (release_id, password) + )`) + this.schema.raw(` + CREATE TABLE spot_state ( + group_name text PRIMARY KEY, + last_article bigint NOT NULL + )`) + } + + async down() { + this.schema.raw(`DROP TABLE IF EXISTS spots, spot_files, release_spots, release_passwords, spot_state`) + } +} diff --git a/deploy/README.md b/deploy/README.md index 974d33a..eb44f00 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -20,6 +20,8 @@ homelab with the deploy key, whose forced command | `coordinator` | `usenet-rebuild` | prefetching (all NNTP connections), planning, the tail, merging, settling | | `prober` | `usenet-rebuild` | phase 2: reads the probe queue's articles over `PROBE_CONNECTIONS` (6) connections, stores what they show | | `merger` | `usenet-rebuild` | the par2 merge: releases whose files one par2 set describes become one (no NNTP) | +| `spotter` | `usenet-rebuild` | Spotnet: free.pt's spots into `spots`, signatures checked, the passwords they name kept; 1 connection | +| `spot-inspector` | `usenet-rebuild` | each spot's NZB: its files in `spot_files`, linked to the releases they match (`release_spots`), passwords in `release_passwords`; 1 connection | | `edge` | `usenet-rebuild-edge` (target `edge`) | the Rust worker, `--threads $EDGE_THREADS` (8) | | `edge-ts` | `usenet-rebuild` | the TypeScript worker, profile `edge-ts`, not started | | `ace` | `usenet-rebuild` | one-off ace commands, profile `tools` | diff --git a/deploy/compose.yaml b/deploy/compose.yaml index 53ade17..27031d2 100644 --- a/deploy/compose.yaml +++ b/deploy/compose.yaml @@ -101,17 +101,18 @@ services: command: ['node', 'ace', 'coordinator:run'] environment: <<: *app-env - NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-40} + NNTP_CONNECTIONS: ${NNTP_CONNECTIONS:-38} # Phase 2's prober: reads the probe queue's articles over PROBE_CONNECTIONS - # connections of its own (next to the coordinator's 40 + 1; the account has 50). + # connections of its own. The account has 50: the coordinator's 38 (+ its + # plain one), the prober's 10, the spotter's and the spot inspector's 1 each. prober: <<: *app command: ['node', 'ace', 'probe:run'] volumes: [] environment: <<: *app-env - PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-6} + PROBE_CONNECTIONS: ${PROBE_CONNECTIONS:-10} # The par2 merge: releases whose files one par2 set describes (what the prober read) become one. No NNTP. merger: @@ -119,6 +120,19 @@ services: command: ['node', 'ace', 'releases:merge'] volumes: [] + # Spotnet: the spotter reads free.pt's spots (verified, with the passwords + # they name) into the spots table; the spot inspector reads each spot's NZB, + # lists its files and links the releases they match. One connection each. + spotter: + <<: *app + command: ['node', 'ace', 'spots:read'] + volumes: [] + + spot-inspector: + <<: *app + command: ['node', 'ace', 'spots:inspect'] + volumes: [] + # The Rust worker; it has no NNTP account and talks to node-api only. # Stopped before node-api (a worker holding a stream keeps the server from # exiting). diff --git a/deploy/deploy.sh b/deploy/deploy.sh index 6ab05be..6df14dd 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -45,10 +45,10 @@ docker compose run --rm --no-deps migrate # stream keeps the node API from exiting. echo "== recreating app services" docker compose stop edge -docker compose up -d --no-deps node-api web coordinator prober merger +docker compose up -d --no-deps node-api web coordinator prober merger spotter spot-inspector docker compose up -d --no-deps edge echo "$SHA $(date -Is)" >> "$ROOT/deployed.log" # The images this replaced (321 MB each; the root filesystem is the NVMe, with a 10 GB floor). docker image prune -f --filter label=com.docker.compose.project=usenet-rebuild >/dev/null -docker compose ps --format '{{.Service}} {{.Status}}' node-api web coordinator prober merger edge +docker compose ps --format '{{.Service}} {{.Status}}' node-api web coordinator prober merger spotter spot-inspector edge echo "== deployed $SHA" diff --git a/lib/nntp/xover_client.ts b/lib/nntp/xover_client.ts index 9db1c71..529ec00 100644 --- a/lib/nntp/xover_client.ts +++ b/lib/nntp/xover_client.ts @@ -246,6 +246,20 @@ export class XoverClient { }) } + /** + * An article's header by message-id, raw (as body(): "\r\n.\r\n" cut off). Spot headers carry the spot's XML + * (X-XML), signed (lib/spotnet). 430 (no such article) throws MissingArticle. + */ + async head(messageId: string): Promise { + return this.timed(async () => { + const id = messageId.startsWith('<') ? messageId : `<${messageId}>` + const reply = await this.command(`HEAD ${id}`) + if (reply.startsWith('430') || reply.startsWith('423')) throw new MissingArticle(reply) + await this.expect(reply, ['221']) + return this.rawBody() + }) + } + /** * Several bodies, pipelined: every BODY is sent at once and the replies read in order, so one round trip * covers them all (one at a time, a connection waited ~0.1–2 s per article: 3–10 articles/s on 6; LOG.md diff --git a/lib/spotnet/categories.ts b/lib/spotnet/categories.ts new file mode 100644 index 0000000..46a3e17 --- /dev/null +++ b/lib/spotnet/categories.ts @@ -0,0 +1,333 @@ +/** + * Spotnet's category tree, from Spotweb's SpotCategories (English names). + * A spot has a head category (0–3) and subcategories per type letter: + * `a` format/platform, `b` source/format/genre, `c` language/bitrate/genre, + * `d` genre, `z` kind (movie, series, book, …). + */ +export const SPOT_HEAD_CATEGORIES = ['Image', 'Sound', 'Games', 'Applications'] + +/** What each type letter means per head category. */ +export const SPOT_SUBCAT_TYPES: Record[] = [ + { a: 'Format', b: 'Source', c: 'Language', d: 'Genre', z: 'Type' }, + { a: 'Format', b: 'Source', c: 'Bitrate', d: 'Genre', z: 'Type' }, + { a: 'Platform', b: 'Format', c: 'Genre', z: 'Type' }, + { a: 'Platform', b: 'Genre', z: 'Type' }, +] + +const NAMES: Record>[] = [ + { + a: { + 0: 'DivX', + 1: 'WMV', + 2: 'MPG', + 3: 'DVD5', + 4: 'HD other', + 5: 'ePub', + 6: 'Blu-ray', + 7: 'HD-DVD', + 8: 'WMVHD', + 9: 'x264', + 10: 'DVD9', + 11: 'PDF', + 12: 'Bitmap', + 13: 'Vector', + 14: '3D', + 15: 'UHD', + }, + b: { + 0: 'CAM', + 1: '(S)VCD', + 2: 'Promo', + 3: 'Retail', + 4: 'TV', + 6: 'Satellite', + 7: 'R5', + 8: 'Telecine', + 9: 'Telesync', + 10: 'Scan', + 11: 'WEB-DL', + 12: 'WEBRip', + 13: 'HDRip', + }, + c: { + 0: 'No subtitles', + 1: 'Dutch subtitles (external)', + 2: 'Dutch subtitles (builtin)', + 3: 'English subtitles (external)', + 4: 'English subtitles (builtin)', + 6: 'Dutch subtitles (available)', + 7: 'English subtitles (available)', + 10: 'English audio/written', + 11: 'Dutch audio/written', + 12: 'German audio/written', + 13: 'French audio/written', + 14: 'Spanish audio/written', + 15: 'Asian audio/written', + }, + d: { + 0: 'Action', + 1: 'Adventure', + 2: 'Animation', + 3: 'Cabaret', + 4: 'Comedy', + 5: 'Crime', + 6: 'Documentary', + 7: 'Drama', + 8: 'Family', + 9: 'Fantasy', + 10: 'Arthouse', + 11: 'Television', + 12: 'Horror', + 13: 'Music', + 14: 'Musical', + 15: 'Mystery', + 16: 'Romance', + 17: 'Science Fiction', + 18: 'Sport', + 19: 'Short movie', + 20: 'Thriller', + 21: 'War', + 22: 'Western', + 23: 'Erotica (hetero)', + 24: 'Erotica (gay male)', + 25: 'Erotica (gay female)', + 26: 'Erotica (bi)', + 28: 'Asian', + 29: 'Anime', + 30: 'Cover', + 31: 'Comicbook', + 32: 'Cartoons', + 33: 'Youth', + 34: 'Business', + 35: 'Computer', + 36: 'Hobby', + 37: 'Cooking', + 38: 'Handwork', + 39: 'Craftwork', + 40: 'Health', + 41: 'History', + 42: 'Psychology', + 43: 'Newspaper', + 44: 'Magazine', + 45: 'Science', + 46: 'Female', + 47: 'Religion', + 48: 'Roman', + 49: 'Biography', + 50: 'Detective', + 51: 'Animals', + 52: 'Humor', + 53: 'Travel', + 54: 'True story', + 55: 'Non-fiction', + 56: 'Politics', + 57: 'Poetry', + 58: 'Fairy tale', + 59: 'Technical', + 60: 'Art', + 72: 'Bi', + 73: 'Lesbian', + 74: 'Homo', + 75: 'Hetero', + 76: 'Amature', + 77: 'Group', + 78: 'POV', + 79: 'Solo', + 80: 'Young', + 81: 'Soft', + 82: 'Fetish', + 83: 'Old', + 84: 'Fat', + 85: 'SM', + 86: 'Rough', + 87: 'Dark', + 88: 'Hentai', + 89: 'Outside', + }, + z: { 0: 'Movie', 1: 'Series', 2: 'Book', 3: 'Erotica', 4: 'Picture' }, + }, + { + a: { + 0: 'MP3', + 1: 'WMA', + 2: 'WAV', + 3: 'OGG', + 4: 'EAC', + 5: 'DTS', + 6: 'AAC', + 7: 'APE', + 8: 'FLAC', + }, + b: { 0: 'CD', 1: 'Radio', 2: 'Compilation', 3: 'DVD', 4: 'Other', 5: 'Vinyl', 6: 'Stream' }, + c: { + 0: 'Variable', + 1: '< 96kbit', + 2: '96kbit', + 3: '128kbit', + 4: '160kbit', + 5: '192kbit', + 6: '256kbit', + 7: '320kbit', + 8: 'Lossless', + 9: 'Other', + }, + d: { + 0: 'Blues', + 1: 'Compilation', + 2: 'Cabaret', + 3: 'Dance', + 4: 'Diverse', + 5: 'Hardcore', + 6: 'World', + 7: 'Jazz', + 8: 'Youth', + 9: 'Classical', + 10: 'Kleinkunst', + 11: 'Dutch', + 12: 'New Age', + 13: 'Pop', + 14: 'RnB', + 15: 'Hiphop', + 16: 'Reggae', + 17: 'Religious', + 18: 'Rock', + 19: 'Soundtracks', + 20: 'Other', + 21: 'Hardstyle', + 22: 'Asian', + 23: 'Disco', + 24: 'Classics', + 25: 'Metal', + 26: 'Country', + 27: 'Dubstep', + 28: 'Nederhop', + 29: 'DnB', + 30: 'Electro', + 31: 'Folk', + 32: 'Soul', + 33: 'Trance', + 34: 'Balkan', + 35: 'Techno', + 36: 'Ambient', + 37: 'Latin', + 38: 'Live', + }, + z: { 0: 'Album', 1: 'Liveset', 2: 'Podcast', 3: 'Audiobook' }, + }, + { + a: { + 0: 'Windows', + 1: 'Macintosh', + 2: 'Linux', + 3: 'Playstation', + 4: 'Playstation 2', + 5: 'PSP', + 6: 'Xbox', + 7: 'Xbox 360', + 8: 'Gameboy Advance', + 9: 'Gamecube', + 10: 'Nintendo DS', + 11: 'Nintento Wii', + 12: 'Playstation 3', + 13: 'Windows Phone', + 14: 'iOS', + 15: 'Android', + 16: 'Nintendo 3DS', + 17: 'Playstation 4', + 18: 'XBox 1', + }, + b: { 0: 'ISO', 1: 'Rip', 2: 'Retail', 3: 'DLC', 4: '', 5: 'Patch', 6: 'Crack' }, + c: { + 0: 'Action', + 1: 'Adventure', + 2: 'Strategy', + 3: 'Roleplaying', + 4: 'Simulation', + 5: 'Race', + 6: 'Flying', + 7: 'Shooter', + 8: 'Platform', + 9: 'Sport', + 10: 'Child/youth', + 11: 'Puzzle', + 12: 'Other', + 13: 'Boardgame', + 14: 'Cards', + 15: 'Education', + 16: 'Music', + 17: 'Family', + }, + }, + { + a: { + 0: 'Windows', + 1: 'Macintosh', + 2: 'Linux', + 3: 'OS/2', + 4: 'Windows Phone', + 5: 'Navigation systems', + 6: 'iOS', + 7: 'Android', + }, + b: { + 0: 'Audio', + 1: 'Video', + 2: 'Graphics', + 3: 'CD/DVD Tools', + 4: 'Media players', + 5: 'Rippers & Encoders', + 6: 'Plugins', + 7: 'Database tools', + 8: 'Email software', + 9: 'Photo', + 10: 'Screensavers', + 11: 'Skin software', + 12: 'Drivers', + 13: 'Browsers', + 14: 'Download managers', + 15: 'Download', + 16: 'Usenet software', + 17: 'RSS Readers', + 18: 'FTP software', + 19: 'Firewalls', + 20: 'Antivirus software', + 21: 'Antispyware software', + 22: 'Optimization software', + 23: 'Security software', + 24: 'System software', + 25: 'Other', + 26: 'Educational', + 27: 'Office', + 28: 'Internet', + 29: 'Communication', + 30: 'Development', + 31: 'Spotnet', + }, + }, +] + +export interface SpotCategoryLabels { + head: string | null + /** In type order (z, a, b, c, d), e.g. { type: 'Language', values: ['Dutch subtitles (builtin)'] }. */ + subcats: { type: string; values: string[] }[] +} + +/** Readable names for a spot's head category and subcategories (e.g. { a: [9], z: [0] }). */ +export function spotCategoryLabels( + category: number | null, + subcats: Record | null +): SpotCategoryLabels { + if (category === null || !(category in NAMES)) return { head: null, subcats: [] } + const labels: SpotCategoryLabels['subcats'] = [] + for (const letter of ['z', 'a', 'b', 'c', 'd']) { + const values = [ + ...new Set( + (subcats?.[letter] ?? []) + .map((n) => NAMES[category][letter]?.[n]) + .filter((v): v is string => Boolean(v)) + ), + ] + if (values.length) labels.push({ type: SPOT_SUBCAT_TYPES[category][letter] ?? letter, values }) + } + return { head: SPOT_HEAD_CATEGORIES[category], subcats: labels } +} diff --git a/lib/spotnet/header.ts b/lib/spotnet/header.ts new file mode 100644 index 0000000..f550483 --- /dev/null +++ b/lib/spotnet/header.ts @@ -0,0 +1,162 @@ +import { createHash, createPublicKey, verify } from 'node:crypto' +import { crc32 } from 'node:zlib' + +/** + * Spotnet spot headers (free.pt), after Spotweb's Services_Format_Parsing::parseHeader. + * + * From: Nick <[PUBKEY[.USERSIGNATURE]]@[CAT][KEYID][SUBCATS].[SIZE].[x].[DATE].[..].[SIGNATURE]> + * Subject: Title | Tag + */ +export interface SpotHeader { + messageId: string + poster: string + title: string + tag: string + /** 0 image/video, 1 sound, 2 games, 3 applications. */ + category: number + keyId: number + /** e.g. { a: [9], b: [3], c: [7, 10], d: [17], z: [0] } */ + subcats: Record + size: number + /** Unix seconds. */ + stamp: number + /** Everything after the @. */ + header: string + headerSign: string + selfSignedPubKey: string + userSignature?: string +} + +/** Spotnet's base64 variant: `-s` for `/`, `-p` for `+`, padding left out. */ +export function spotUnprepareBase64(input: string): string { + const padded = input.length % 4 ? input + '='.repeat(4 - (input.length % 4)) : input + return padded.replace(/-s/g, '/').replace(/-p/g, '+') +} + +/** Parses the From/Subject of a spot. Null when it isn't one. */ +export function parseSpotHeader(from: string, subject: string, messageId: string, date?: string): SpotHeader | null { + const lt = from.indexOf('<') + if (lt === -1) return null + const address = from.slice(lt + 1, from.endsWith('>') ? -1 : undefined) + const at = address.lastIndexOf('@') + if (at === -1) return null + const header = address.slice(at + 1) + const [pubKey, userSignature] = address.slice(0, at).split('.') + + const fields = header.split('.') + if (fields.length < 6) return null + const category = Number.parseInt(fields[0][0], 10) - 1 + const keyId = Number.parseInt(fields[0][1], 10) + if (!(category >= 0) || !(keyId >= 0)) return null + + // a9b4c0d5 (FTD) or a09b04c00d05 (newer): a letter, then digits. + const subcats: Record = {} + for (const [, letter, digits] of fields[0].slice(2).toLowerCase().matchAll(/([a-z])(\d+)/g)) { + if ('abcdz'.includes(letter)) (subcats[letter] ??= []).push(Number.parseInt(digits, 10)) + } + + // Newer keys: "Title | Tag". Key 1 (oldest): the tag is the last field. + const parts = subject.split('|') + let title: string + let tag = '' + if (keyId !== 1) { + title = parts[0].trim() + tag = parts.length > 1 ? parts[1].trim() : '' + } else { + tag = parts.length > 1 ? parts[parts.length - 1].trim() : subject.trim() + title = parts.length > 1 ? parts.slice(0, -2).join('|').trim() : '' + } + + const parsedDate = date ? Date.parse(date) : Number.NaN + return { + messageId: messageId.replace(/^<|>$/g, ''), + poster: from.slice(0, lt).trim(), + title, + tag, + category, + keyId, + subcats, + size: Number.parseInt(fields[1], 10) || 0, + stamp: Number.isNaN(parsedDate) ? Number.parseInt(fields[3], 10) || 0 : Math.floor(parsedDate / 1000), + header, + headerSign: fields[fields.length - 1], + selfSignedPubKey: spotUnprepareBase64(pubKey ?? ''), + userSignature: userSignature ? spotUnprepareBase64(userSignature) : undefined, + } +} + +/** Spotnet's own signing keys (Spotweb settings `rsa_keys`), base64 modulus. */ +export const SPOTNET_KEYS: Record = { + 2: 'ys8WSlqonQMWT8ubG0tAA2Q07P36E+CJmb875wSR1XH7IFhEi0CCwlUzNqBFhC+P', + 3: 'uiyChPV23eguLAJNttC/o0nAsxXgdjtvUvidV2JL+hjNzc4Tc/PPo2JdYvsqUsat', + 4: '1k6RNDVD6yBYWR6kHmwzmSud7JkNV4SMigBrs+jFgOK5Ldzwl17mKXJhl+su/GR9', +} + +const base64url = (b64: string) => Buffer.from(b64, 'base64').toString('base64url') + +/** RSA PKCS#1 v1.5 with SHA-1, as Spotweb's checkRsaSignature (exponent AQAB). */ +export function checkRsaSignature(message: Buffer | string, signatureB64: string, modulusB64: string): boolean { + try { + const key = createPublicKey({ key: { kty: 'RSA', n: base64url(modulusB64), e: 'AQAB' }, format: 'jwk' }) + const data = typeof message === 'string' ? toBytes(message) : message + return verify('sha1', data, key, Buffer.from(signatureB64, 'base64')) + } catch { + return false + } +} + +/** Spots are signed over their ISO-8859-1 bytes. */ +function toBytes(text: string): Buffer { + return /^[\u0000-ÿ]*$/.test(text) ? Buffer.from(text, 'latin1') : Buffer.from(text, 'utf8') +} + +/** SPOTSIGN_V2 and self-signed messages carry a hashcash: sha1('') starts with 0000. */ +function hashcash(messageId: string): boolean { + return createHash('sha1').update(`<${messageId}>`).digest('hex').startsWith('0000') +} + +/** + * Verifies a spot header like Spotweb does: key 7 (self-signed) needs a + * hashcash and the poster's key, keys 2–4 Spotnet's keys; spots that also + * carry a user signature must have it match the message id. + */ +export function verifySpotHeader(spot: SpotHeader, keys: Record = SPOTNET_KEYS): boolean { + if (!spot.title || !spot.poster) return false + // Old key 1 spots from before 2011 weren't signed. + if (spot.keyId === 1 && spot.stamp <= 1293870080) return true + if (!spot.headerSign) return false + + const signature = spotUnprepareBase64(spot.headerSign) + const toCheck = spot.title + spot.header.slice(0, spot.header.length - spot.headerSign.length - 1) + spot.poster + let verified: boolean + if (spot.keyId === 7 || (spot.keyId === 2 && spot.selfSignedPubKey.length > 50)) { + // Self-signed (and key-2 "dispose" messages): the hashcash is the gate, as in Spotweb. + verified = hashcash(spot.messageId) + } else { + verified = keys[spot.keyId] !== undefined && checkRsaSignature(toCheck, signature, keys[spot.keyId]) + } + if (verified && spot.userSignature && spot.selfSignedPubKey) { + verified = checkRsaSignature(`<${spot.messageId}>`, spot.userSignature, spot.selfSignedPubKey) + } + return verified +} + +/** Spotweb's spotter id: base64 of the CRC32 (little-endian) of the poster's public key. */ +export function spotterId(pubKeyB64: string): string { + if (!pubKeyB64) return '' + const crc = Buffer.alloc(4) + crc.writeUInt32LE(crc32(Buffer.from(pubKeyB64, 'base64')) >>> 0) + return crc.toString('base64').replace(/[/+=]/g, '') +} + +/** + * Moderation: key-2 spots titled "delete|dispose|remove ". + * From Spotnet's key they are moderator actions; with the poster's own key + * (a spotter id) a personal dispose of their own spot. + */ +export function moderationTarget(spot: Pick): string | null { + if (spot.keyId !== 2) return null + const [command, target] = spot.title.split(' ') + if (!['delete', 'dispose', 'remove'].includes(command?.toLowerCase()) || !target) return null + return target.replace(/^<|>$/g, '') +} diff --git a/lib/spotnet/index.ts b/lib/spotnet/index.ts new file mode 100644 index 0000000..fb6c852 --- /dev/null +++ b/lib/spotnet/index.ts @@ -0,0 +1,9 @@ +/** + * Spotnet (free.pt) spots, independent of AdonisJS: header parsing and + * signature checks after Spotweb, the spot XML and its compressed NZB. + * See CONCEPT.md (D29). + */ +export { parseSpotHeader, verifySpotHeader, spotterId, moderationTarget, checkRsaSignature, spotUnprepareBase64, SPOTNET_KEYS, type SpotHeader } from './header.js' +export { parseSpotXml, parseSpotArticleHeaders, decodeSpotBinary, decodeSpotText, type SpotXml, type SpotImage } from './xml.js' +export { spotCategoryLabels, SPOT_HEAD_CATEGORIES, type SpotCategoryLabels } from './categories.js' +export { passwordFromSpot } from './password.js' diff --git a/lib/spotnet/password.ts b/lib/spotnet/password.ts new file mode 100644 index 0000000..4247254 --- /dev/null +++ b/lib/spotnet/password.ts @@ -0,0 +1,83 @@ +/** + * Archive passwords that spotters put in the title or description instead of + * the NZB's ``: + * `Title{{secret}}` (the newznab/SABnzbd convention), + * `Wachtwoord: KKKK2`, `[b]Password[/b]: x`, `WW: p3IU5Rz9R`, `ww= 0112358`, + * a few words between keyword and separator (`Het wachtwoord voor de nzb is: dw0067`, + * `ww in nzb: X`, `Wachtwoord inbegrepen: wysiwyg`), + * the value on the next line (`WW:` / `tijdloos`), + * `wachtwoord is arn2`, `… voorzien van een wachtwoord. Deze is: X`, + * `WW FC0401` (abbreviation without separator: with a digit in it, or as the whole line). + * The value must be the rest of its line and a single token, so sentences + * such as "Wachtwoord: deze download is zonder wachtwoord geplaatst" or + * "met een password beschermd" don't count (LOG.md 2026-10-08 (10)). + */ +const KEYWORD = String.raw`(?:wachtwoord|wachtw|password|passwort|passwd|pwd|pass|ww|pw)(?![a-z])` +const WITH_SEPARATOR = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}(?:\s+[a-zà-ÿ]+){0,3}?\s*(?:is\s*)?[:=]\s*(.*)$`, 'i') +const IS_VALUE = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}\s+is\s+(\S+)\s*$`, 'i') +/** "Deze is voorzien van een wachtwoord. Deze is: X": the value in the next sentence. */ +const NEXT_SENTENCE = /(?:^|[.!?]\s*)(?:deze|dit|het|die|this|it)\s+is\s*[:=]\s*(\S+)\s*$/i +const MENTIONS = new RegExp(String.raw`(?:^|[^a-z0-9])${KEYWORD}`, 'i') +const ABBREVIATION = /(?:^|[^a-z0-9])(?:ww|pw)\s+(\S+)\s*$/i +/** "WW carrie" as a line of its own; "WW II" is the war. */ +const WHOLE_LINE_ABBREVIATION = /^(?:ww|pw)\s+(\S+)$/i +const NOT_PASSWORD_TOKEN = /^(?:i{1,3}|iv|v|[12])$/i +const NO_PASSWORD = /\b(zonder|geen|no|without|none|niet|kein|ohne)\b.{0,20}\b(wachtwoord|password|passwort|ww|pw)\b|\b(wachtwoord|password|passwort)\b.{0,20}\b(niet nodig|not needed|none|geen|nvt|n\/a)\b/i +const NOT_A_VALUE = /^(?:-+|geen|none|no|nee|nvt|n\/a|niet|not|leeg|empty|zonder|without|onbekend|unknown|zie|see|in|staat|volgt|nzb|spot)$/i + +/** Lines of a spot description: UBB/HTML line breaks split, other tags dropped. */ +function lines(description: string): string[] { + return description + .replace(/\[br\]|/gi, '\n') + .replace(/\[\/?[a-z]+(?:=[^\]]*)?\]|<\/?[a-z][^>]*>/gi, '') + .split(/\r?\n/) + .map((line) => line.trim()) +} + +function clean(value: string): string | null { + const token = value + .trim() + // "Wachtwoord : yakowyke !!" + .replace(/^[!*~#]+\s*|\s*[!*~#]+$/g, '') + .replace(/^["'“‘«(<]+|["'”’»)>.,;!]+$/g, '') + if (!token || /\s/.test(token) || token.length < 2 || token.length > 64 || NOT_A_VALUE.test(token)) return null + return token +} + +export function passwordFromSpot(title: string, description?: string | null): string | null { + const braces = /\{\{([^{}]{1,64})\}\}/.exec(title) + if (braces && clean(braces[1])) return clean(braces[1]) + const all = lines(description ?? '') + for (const [i, line] of all.entries()) { + if (!line || NO_PASSWORD.test(line)) continue + const separated = WITH_SEPARATOR.exec(line) + if (separated) { + const value = separated[1].trim() ? clean(separated[1]) : nextLineValue(all, i) + if (value) return value + continue + } + const is = IS_VALUE.exec(line) + if (is) { + const value = clean(is[1]) + if (value) return value + } + const sentence = MENTIONS.test(line) ? NEXT_SENTENCE.exec(line) : null + if (sentence) { + const value = clean(sentence[1]) + if (value) return value + } + const whole = WHOLE_LINE_ABBREVIATION.exec(line) + const abbreviated = whole ?? ABBREVIATION.exec(line) + if (abbreviated && (whole ? !NOT_PASSWORD_TOKEN.test(abbreviated[1]) : /\d/.test(abbreviated[1]))) { + const value = clean(abbreviated[1]) + if (value) return value + } + } + return null +} + +/** `WW:` with the value alone on the next non-empty line. */ +function nextLineValue(all: string[], i: number): string | null { + const next = all.slice(i + 1).find((line) => line !== '') + return next ? clean(next) : null +} diff --git a/lib/spotnet/xml.ts b/lib/spotnet/xml.ts new file mode 100644 index 0000000..3a7de79 --- /dev/null +++ b/lib/spotnet/xml.ts @@ -0,0 +1,131 @@ +import { inflateRawSync } from 'node:zlib' + +export interface SpotXml { + title: string + category: number | null + subcats: string[] + size: number + website: string + /** Spotnet markup: UBB tags such as [br], [b], [url=…]…[/url]. */ + description: string + tag: string + /** Newer spots: the image posted as articles (uncompressed); older ones: a web URL. */ + image: SpotImage | null + /** Message ids (without <>) of the compressed NZB in alt.binaries.ftd. */ + nzbSegments: string[] + created: number | null +} + +export type SpotImage = { segments: string[]; width?: number; height?: number } | { url: string } + +const ENTITIES: Record = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'" } + +/** Element text: CDATA unwrapped, entities decoded (also inside CDATA, as Spotweb's html_entity_decode does). */ +function text(value: string | undefined): string { + if (value === undefined) return '' + const cdata = /^\s*\s*$/.exec(value) + return (cdata ? cdata[1] : value).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (match, entity: string) => { + if (entity[0] === '#') { + const code = entity[1].toLowerCase() === 'x' ? Number.parseInt(entity.slice(2), 16) : Number.parseInt(entity.slice(1), 10) + return Number.isFinite(code) && code > 0 && code <= 0x10ffff ? String.fromCodePoint(code) : match + } + return ENTITIES[entity.toLowerCase()] ?? match + }) +} + +const element = (xml: string, name: string) => new RegExp(`<${name}(?:\\s[^>]*)?>([\\s\\S]*?)`, 'i').exec(xml)?.[1] + +/** The spot XML from the X-XML header lines (a tolerant scanner, as spots are often invalid XML). */ +export function parseSpotXml(xml: string): SpotXml | null { + const posting = element(xml, 'Posting') + if (!posting) return null + const categoryXml = element(posting, 'Category') ?? '' + const key = Number.parseInt(element(posting, 'Key') ?? '', 10) + const rawCategory = Number.parseInt(categoryXml.replace(/<[\s\S]*$/, ''), 10) + const nzb = element(posting, 'NZB') ?? '' + return { + description: text(element(posting, 'Description')).trim(), + tag: text(element(posting, 'Tag')).trim(), + image: parseImage(posting), + title: text(element(posting, 'Title')).trim(), + // Newer keys number categories from 1. + category: Number.isNaN(rawCategory) ? null : key !== 1 ? rawCategory - 1 : rawCategory, + subcats: [...categoryXml.matchAll(/<(?:Sub|SubCat)>([^<]*)<\/(?:Sub|SubCat)>/gi)].map((m) => m[1].trim()), + size: Number.parseInt(element(posting, 'Size') ?? '', 10) || 0, + website: text(element(posting, 'Website')).trim(), + nzbSegments: [...nzb.matchAll(/([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean), + created: Number.parseInt(element(posting, 'Created') ?? '', 10) || null, + } +} + +function parseImage(posting: string): SpotImage | null { + const match = /]*)?>([\s\S]*?)<\/Image>/i.exec(posting) + if (!match) return null + const segments = [...match[2].matchAll(/([^<>]*)<\/Segment>/gi)].map((m) => m[1].trim()).filter(Boolean) + if (segments.length > 0) { + const size = (name: string) => Number.parseInt(new RegExp(`${name}=['"]?(\\d+)`, 'i').exec(match[1] ?? '')?.[1] ?? '', 10) || undefined + return { segments, width: size('Width'), height: size('Height') } + } + const url = text(match[2]).trim() + return /^https?:\/\//i.test(url) ? { url } : null +} + +/** + * The X-XML as text. Spotnet clients write UTF-8, very old ones ISO-8859-1; + * the header is read as ISO-8859-1 bytes, so re-decode when it's valid UTF-8. + */ +export function decodeSpotText(latin1: string): string { + const utf8 = Buffer.from(latin1, 'latin1').toString('utf8') + return utf8.includes('\uFFFD') ? latin1 : utf8 +} + +/** + * Header lines of a raw HEAD response; X-XML (split over many lines) joined. + * Folded headers are unfolded: old (key 1) spots put the From address on a + * continuation line. + */ +export function parseSpotArticleHeaders(raw: string): { headers: Record; xml: string } { + const headers: Record = {} + let xml = '' + let last: string | null = null + let lastIsFirst = false + for (const line of raw.split(/\r?\n/)) { + if (/^[ \t]/.test(line) && last) { + if (last === 'x-xml') xml += line + else if (lastIsFirst) headers[last] += line + continue + } + const colon = line.indexOf(':') + if (colon <= 0) { + last = null + continue + } + const name = line.slice(0, colon).toLowerCase() + const value = line.slice(colon + 1).replace(/^ /, '') + last = name + lastIsFirst = !(name in headers) + if (name === 'x-xml') xml += value + else headers[name] ??= value + } + return { headers, xml } +} + +/** + * Spot binaries (the NZB, images): article bodies joined without line + * breaks, Spotnet's escapes undone (=A NUL, =B CR, =C LF, =D =), then raw + * deflate (PHP gzinflate). + */ +export function decodeSpotBinary(bodies: Buffer[], compressed = true): Buffer { + const joined = bodies + .map((body) => + body + .toString('latin1') + .split('\r\n') + // NNTP dot-stuffing. + .map((line) => (line.startsWith('..') ? line.slice(1) : line)) + .join('') + ) + .join('') + const unescaped = Buffer.from(joined.replace(/=([ABCD])/g, (_, c: string) => ({ A: '\0', B: '\r', C: '\n', D: '=' })[c]!), 'latin1') + return compressed ? inflateRawSync(unescaped) : unescaped +} diff --git a/start/env.ts b/start/env.ts index 3754d48..ed2cb0c 100644 --- a/start/env.ts +++ b/start/env.ts @@ -61,6 +61,11 @@ export default await Env.create(new URL('../', import.meta.url), { LEASE_TTL_SECONDS: Env.schema.number.optional(), SPOOL_DIR: Env.schema.string.optional(), CORPUS_DIR: Env.schema.string.optional(), + SPOT_GROUP: Env.schema.string.optional(), + SPOT_BACKFILL: Env.schema.number.optional(), + SPOT_EVERY_SECONDS: Env.schema.number.optional(), + SPOT_MAX_NZB_SEGMENTS: Env.schema.number.optional(), + SPOT_STANDALONE_AFTER_HOURS: Env.schema.number.optional(), CORPUS_AHEAD: Env.schema.number.optional(), PROBE_PIPELINE: Env.schema.number.optional(), SPOOL_AHEAD_PER_LEASE: Env.schema.number.optional(), diff --git a/tests/functional/spots.spec.ts b/tests/functional/spots.spec.ts new file mode 100644 index 0000000..a2b6126 --- /dev/null +++ b/tests/functional/spots.spec.ts @@ -0,0 +1,109 @@ +import { test } from '@japa/runner' +import db from '@adonisjs/lucid/services/db' +import testUtils from '@adonisjs/core/services/test_utils' +import { Spotter } from '#services/spotter' +import { SpotInspector } from '#services/spot_inspector' +import { releaseIdentity } from '#lib/release/index' +import { parseSubject } from '#lib/headers/index' + +// Message ids whose sha1('') starts with 0000: self-signed spots (key 7) pass on that hashcash alone. +const SPOT = 'spottest731@spot.net' +const DISPOSE = 'spottest168100@spot.net' +const OTHER = 'spottest173225@spot.net' +// A poster's key longer than 50 characters: its spotter id makes a personal dispose count. +const PUBKEY = 'A'.repeat(64) +const STAMP = 1_760_000_000 + +/** A spot's raw HEAD: From with the spot header (category 1, key 7), the X-XML with title, description and NZB. */ +function rawSpot(messageId: string, title: string, description: string, key = 7, stamp = STAMP) { + return [ + `From: Spotter <${PUBKEY}@1${key}a09z00.734003200.0.${stamp}.0.SIG>`, + `Subject: ${title} | Tag`, + `Message-ID: <${messageId}>`, + `X-XML: ${key}${title}${description}`, + `X-XML: 0101a09nzb1@spot.net734003200`, + '', + ].join('\r\n') +} + +/** An NZB as spotters post them: the files' original subjects (buildNzb writes its own). */ +function nzbXml(subjects: string[], poster: string, date: number, password?: string) { + const esc = (t: string) => t.replace(/&/g, '&').replace(/"/g, '"').replace(//g, '>') + const head = password ? `${esc(password)}` : '' + const files = subjects + .map((s, i) => `alt.binaries.test.spotsseg${i}@x`) + .join('') + return `${head}${files}` +} + +test.group('Spots', (group) => { + group.each.setup(() => testUtils.db().withGlobalTransaction()) + + test('the spotter lists a verified spot with the password its text names, and a dispose removes it', async ({ assert }) => { + const spotter = new Spotter() + assert.equal(await spotter.store(rawSpot(SPOT, 'Some.Movie.2024.1080p', 'Mooie film[br]Wachtwoord: geheim123'), SPOT, 10), 'listed') + const spot = await db.from('spots').where('message_id', SPOT).firstOrFail() + assert.equal(spot.title, 'Some.Movie.2024.1080p') + assert.equal(spot.password, 'geheim123') + assert.equal(spot.password_source, 'spot-text') + assert.deepEqual(spot.nzb_segments, ['nzb1@spot.net']) + // A {{pw}} in the title wins, and is said to come from the title. + assert.equal(await spotter.store(rawSpot(OTHER, 'Other.Show{{t1tl3pw}}', 'Geen ww'), OTHER, 11), 'listed') + assert.equal((await db.from('spots').where('message_id', OTHER).firstOrFail()).password_source, 'spot-title') + // A message id without the hashcash doesn't verify. + assert.equal(await spotter.store(rawSpot('nohash@spot.net', 'X', ''), 'nohash@spot.net', 12), 'unverified') + // The poster's own dispose within 5 days. + await db.table('release_spots').insert({ release_id: 1, spot_id: SPOT, files: 3 }) + assert.equal(await spotter.store(rawSpot(DISPOSE, `dispose <${SPOT}>`, '', 2, STAMP + 3600), DISPOSE, 13), 'moderation') + assert.equal((await db.from('spots').where('message_id', SPOT).firstOrFail()).status, 'removed') + assert.equal((await db.from('release_spots').where('spot_id', SPOT).select('*')).length, 0) + }) + + test("the inspector lists an NZB's files and links the releases they match, with the password apart", async ({ assert }) => { + const [g] = await db.table('groups').insert({ name: 'alt.binaries.test.spots', priority: 1 }).returning('id') + const groupId = Number(g.id) + const poster = 'up ' + const posted = new Date('2025-10-09T10:00:00Z') + const subjects = [1, 2, 3].map((i) => `Some.Movie.2024.1080p [${i}/3] - "Some.Movie.2024.1080p.part0${i}.rar" yEnc (1/50)`) + // The release the header pass built for these files: the same identity, in the group. + const parsed = parseSubject(subjects[0])! + const identity = releaseIdentity({ poster, subject: parsed.base, name: parsed.name, weak: false })! + const [r] = await db + .table('releases') + .insert({ release_key: identity.key, identity_key: identity.key, name: identity.name, source: identity.source, poster, groups: '[]', group_id: groupId, files: 3, posted_at: posted, readable: true }) + .returning('id') + const releaseId = Number(r.id) + const nzb = nzbXml(subjects, poster, posted.getTime() / 1000, 'nzbpw') + await new Spotter().store(rawSpot(SPOT, 'Some.Movie.2024.1080p', 'Wachtwoord: geheim123'), SPOT, 10) + const spot = await db.from('spots').where('message_id', SPOT).firstOrFail() + + const outcome = await new SpotInspector().inspectNzb(spot, nzb) + assert.deepEqual(outcome, { status: 'inspected', matched: 1 }) + const files = await db.from('spot_files').where('spot_id', SPOT).orderBy('idx') + assert.deepEqual(files.map((f) => f.name), ['Some.Movie.2024.1080p.part01.rar', 'Some.Movie.2024.1080p.part02.rar', 'Some.Movie.2024.1080p.part03.rar']) + assert.deepEqual([...new Set(files.map((f) => Number(f.release_id)))], [releaseId]) + assert.equal((await db.from('release_spots').where({ release_id: releaseId, spot_id: SPOT }).firstOrFail()).files, 3) + // The NZB's own password over the one in the spot's text, kept apart from the release. + const passwords = await db.from('release_passwords').where('release_id', releaseId).select('password', 'source') + assert.deepEqual(passwords, [{ password: 'nzbpw', source: 'nzb' }]) + assert.isTrue((await db.from('releases').where('id', releaseId).firstOrFail()).passworded) + const after = await db.from('spots').where('message_id', SPOT).firstOrFail() + assert.equal(after.status, 'inspected') + assert.equal(Number(after.release_id), releaseId) + assert.equal(after.matched, 3) + }) + + test('a spot whose files match no release becomes one of its own, with its NZB', async ({ assert }) => { + await db.table('groups').insert({ name: 'alt.binaries.test.spots', priority: 1 }) + const nzb = nzbXml(['[1/1] - "lonely.file.mkv" yEnc (1/9)'], 'p', STAMP) + await new Spotter().store(rawSpot(SPOT, 'Lonely File', ''), SPOT, 10) + const inspector = new SpotInspector() + assert.deepEqual(await inspector.inspectNzb(await db.from('spots').where('message_id', SPOT).firstOrFail(), nzb), { status: 'inspected', matched: 0 }) + assert.isTrue(await inspector.standalone(SPOT)) + const release = await db.from('releases').where('source', 'spot').firstOrFail() + assert.equal(release.name, 'Lonely File') + assert.equal(release.files, 1) + const { default: NzbService } = await import('#services/nzb_service') + assert.include((await NzbService.build(Number(release.id)))!.xml, 'lonely.file.mkv') + }) +}) diff --git a/tests/unit/spotnet.spec.ts b/tests/unit/spotnet.spec.ts new file mode 100644 index 0000000..5ba991f --- /dev/null +++ b/tests/unit/spotnet.spec.ts @@ -0,0 +1,186 @@ +import { test } from '@japa/runner' +import { deflateRawSync } from 'node:zlib' +import { + decodeSpotBinary, + decodeSpotText, + parseSpotArticleHeaders, + parseSpotHeader, + parseSpotXml, + spotCategoryLabels, + verifySpotHeader, +} from '#lib/spotnet/index' +import { ubbToText } from '#lib/newznab/xml' + +// A real spot from free.pt (self-signed, key 7), 2026-09-28. +const SPOT = {"from": "Bart <0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE-s9x.Fbc6Dm09v32bUgXpDT88QADBHELZ7wP4sMIMHKpQYxBIlK-pZveUFu0aMYUx-pUf6Z@17a09b03c07c10d17z00.21651233204.20.1790619886.1.NL.Uyf3JHAAlVvgAg9HKIu3veFDGC4DX9-sgTnchacVAkC-p3USalOe-sacdamKdgXirL6>", "subject": "The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi | Bart", "id": "", "date": "Mon, 28 Sep 2026 18:24:45 GMT"} + +test.group('spotnet / headers', () => { + test('parses a spot header', ({ assert }) => { + const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)! + assert.include(spot, { + poster: 'Bart', + title: 'The.Incredible.Shrinking.Woman.1981.1080p.BluRay.REMUX.AVC.FLAC.2.0-MuGi', + tag: 'Bart', + category: 0, + keyId: 7, + size: 21651233204, + messageId: 'jWCbaAa4f1A7LC6agmOEx@spot.net', + }) + assert.deepEqual(spot.subcats, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] }) + }) + + test('verifies its signatures, and rejects another message id', ({ assert }) => { + const spot = parseSpotHeader(SPOT.from, SPOT.subject, SPOT.id, SPOT.date)! + assert.isTrue(verifySpotHeader(spot)) + assert.isFalse(verifySpotHeader({ ...spot, messageId: 'other@spot.net' })) + assert.isNull(parseSpotHeader('someone ', 'subject', '')) + }) +}) + +test.group('spotnet / xml and nzb', () => { + test('reads the NZB segments and category from the X-XML lines', ({ assert }) => { + const raw = [ + 'From: Bart ', + 'X-XML: 7A & B0101a09', + 'X-XML: 01z00abc@spot.netdef@spot.net', + 'X-XML: 123', + ].join('\r\n') + const { headers, xml } = parseSpotArticleHeaders(raw) + assert.equal(headers.from, 'Bart ') + const spot = parseSpotXml(xml)! + assert.deepEqual(spot.nzbSegments, ['abc@spot.net', 'def@spot.net']) + assert.include(spot, { title: 'A & B', category: 0, size: 123 }) + assert.deepEqual(spot.subcats, ['01a09', '01z00']) + }) + + test('reads the description, tag and image', ({ assert }) => { + const spot = parseSpotXml( + `7BartT` + + `Lily Tomlin[br][br]If you're ‘in’ [url=https://example.com/a]here[/url]` + + `FYe6l94kQsY7LC6agh5Xn@spot.net` + + `0101a09` + )! + assert.equal(spot.description, 'Lily Tomlin[br][br]If you\'re \u2018in\u2019 [url=https://example.com/a]here[/url]') + assert.equal(spot.tag, 'Bart') + assert.deepEqual(spot.image, { segments: ['FYe6l94kQsY7LC6agh5Xn@spot.net'], width: 185, height: 278 }) + // Older spots link an image on the web. + assert.deepEqual(parseSpotXml('http://x.nl/a.jpg')!.image, { url: 'http://x.nl/a.jpg' }) + assert.isNull(parseSpotXml('T')!.image) + }) + + test('reads old (key 1) spots: a folded From line, HTML in CDATA', ({ assert }) => { + const raw = [ + 'From: vanker', + ' <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>', + 'Subject: The postman | The postman |', + 'X-XML: The postmanVan "The', + 'X-XML: postman"]]>http://covertarget.com/si.php?id=100', + 'Date: 01 Feb 2010 02:43:16 GMT', + ].join('\r\n') + const { headers, xml } = parseSpotArticleHeaders(raw) + assert.equal(headers.from, 'vanker <1100095@11a10b5c5d0d1.g30.1100095.1256142174.T6tl.0.rm0wwOrdI-sVDC-pXXRIXJZdxX5cvNZy80ro7LQBumtv2b-pLYpG5RZ3g>') + assert.equal(headers.date, '01 Feb 2010 02:43:16 GMT') + const spot = parseSpotXml(xml)! + assert.equal(spot.description, 'post voor de lol.
Van "The postman"') + assert.deepEqual(spot.image, { url: 'http://covertarget.com/si.php?id=1' }) + assert.equal(ubbToText(spot.description), 'post voor de lol.\nVan "The postman"') + }) + + test('reads header text as UTF-8 when it is, else as ISO-8859-1', ({ assert }) => { + assert.equal(decodeSpotText(Buffer.from('Café', 'utf8').toString('latin1')), 'Café') + assert.equal(decodeSpotText(Buffer.from('Café', 'latin1').toString('latin1')), 'Café') + }) + + test('names categories as Spotweb does', ({ assert }) => { + assert.deepEqual(spotCategoryLabels(0, { a: [9], b: [3], c: [7, 10], d: [17], z: [0] }), { + head: 'Image', + subcats: [ + { type: 'Type', values: ['Movie'] }, + { type: 'Format', values: ['x264'] }, + { type: 'Source', values: ['Retail'] }, + { type: 'Language', values: ['English subtitles (available)', 'English audio/written'] }, + { type: 'Genre', values: ['Science Fiction'] }, + ], + }) + assert.deepEqual(spotCategoryLabels(null, null), { head: null, subcats: [] }) + }) + + test('turns UBB markup into plain text', ({ assert }) => { + assert.equal(ubbToText('A[br][b]B[/b] [url=https://x.nl]site[/url][br]'), 'A\nB site (https://x.nl)') + }) + + test('decodes an escaped, deflated NZB split over articles', ({ assert }) => { + const nzb = '\n\r\n=\0'.repeat(20) + const escaped = deflateRawSync(Buffer.from(nzb, 'latin1')) + .toString('latin1') + .replace(/[=\n\r\0]/g, (c) => ({ '=': '=D', '\n': '=C', '\r': '=B', '\0': '=A' })[c]!) + // Posted as lines, dot-stuffed, over two articles. + const lines = escaped.match(/.{1,40}/gs)!.map((l) => (l.startsWith('.') ? '.' + l : l)) + const half = Math.ceil(lines.length / 2) + const bodies = [lines.slice(0, half), lines.slice(half)].map((ls) => Buffer.from(ls.join('\r\n'), 'latin1')) + assert.equal(decodeSpotBinary(bodies).toString('latin1'), nzb) + }) +}) + +test.group('spotnet / moderation', () => { + test('recognises delete and dispose commands from key 2 only', async ({ assert }) => { + const { moderationTarget, spotterId } = await import('#lib/spotnet/index') + assert.equal(moderationTarget({ keyId: 2, title: 'DISPOSE 7IzfA0aCFTkOne6agvY06@spot.net - The Voice DE S15E07 (1080p)' }), '7IzfA0aCFTkOne6agvY06@spot.net') + assert.equal(moderationTarget({ keyId: 2, title: 'delete ' }), 'abc@spot.net') + assert.isNull(moderationTarget({ keyId: 7, title: 'DISPOSE abc@spot.net' })) + assert.isNull(moderationTarget({ keyId: 2, title: 'The Disposal 2024' })) + // Spotweb: base64 of the little-endian CRC32 of the key, without / + =. + assert.match(spotterId('0RnzK11s9B4jqZdfc0UHj1qpSrgBw7nKaYBPckvJuk8sKchVXvdofY6qcKkZE/9x'), /^[A-Za-z0-9]{6}$/) + }) +}) + +test.group('spotnet / passwords in descriptions', () => { + test('finds passwords written in the description or title', async ({ assert }) => { + const { passwordFromSpot } = await import('#lib/spotnet/index') + assert.equal(passwordFromSpot('T', 'Language: NL[br][br][b]Wachtwoord[/b]: KKKK2[br][br]Heb je spotnet?'), 'KKKK2') + assert.equal(passwordFromSpot('T', '===[br][br]AV1 BRRip[br][br]WW: p3IU5Rz9R'), 'p3IU5Rz9R') + assert.equal(passwordFromSpot('T', 'Have fun[br]ww= 011235813[br]Staat er over een half uur wel op.'), '011235813') + assert.equal(passwordFromSpot('T', 'problemen begonnen.[br]WW FC0401[br][br]thx'), 'FC0401') + assert.equal(passwordFromSpot('T', 'mysteries.[br][br]Wachtwoord: Wololoo[br](het kan zijn dat'), 'Wololoo') + assert.equal(passwordFromSpot('Some.Movie.2020{{s3cr3t}}', ''), 's3cr3t') + assert.equal(passwordFromSpot('T', '
Password = "abc123"
'), 'abc123') + }) + + test('finds passwords in the phrasings found in spots that the first version missed', async ({ assert }) => { + const { passwordFromSpot } = await import('#lib/spotnet/index') + // LOG.md 2026-10-08 (10) + assert.equal(passwordFromSpot('T', 'Formaat: DVD5 ISO[br][br]!! LETOP Wachtwoord : oscfyajg !!'), 'oscfyajg') + assert.equal(passwordFromSpot('T', "Het totaal gewicht komt neer op zo'n 400GB. Deze is voorzien van een wachtwoord. Deze is: VTSm7nLdZZl5Kren[br]"), 'VTSm7nLdZZl5Kren') + assert.equal(passwordFromSpot('T', 'Futurama S11E10 1080P[br][br]ww in nzb: cI6f7dBV6BKkhqD[br][br]Dit was'), 'cI6f7dBV6BKkhqD') + assert.equal(passwordFromSpot('T', '[b]Het wachtwoord voor de nzb is: dw0022[/b][br]'), 'dw0022') + assert.equal(passwordFromSpot('T', 'haakinstructies.[br][br]Wachtwoord inbegrepen: wysiwyg'), 'wysiwyg') + assert.equal(passwordFromSpot('T', 'afgeslagen moeten worden.[br][br]wachtwoord is arn2'), 'arn2') + assert.equal(passwordFromSpot('T', 'Forced : No[br][br]WW:[br]tijdloos[br][br]Zag en merkte'), 'tijdloos') + assert.equal(passwordFromSpot('T', 'telekinetische krachten beschikt.[br]WW carrie[br][br]Multisubs'), 'carrie') + }) + + test('ignores sentences about passwords', async ({ assert }) => { + const { passwordFromSpot } = await import('#lib/spotnet/index') + for (const description of [ + 'selecteerbaar.[br]Wachtwoord: deze download is zonder wachtwoord geplaatst.[br]', + '[b]--> Geen wachtwoord benodigd [/b]', + 'zitten 2 bestanden in en is met een password beschermd.', + 'It also removes the need for manual password entry and improves the reliability', + 'Codeermodus: Hoge kwaliteit (2-Pass VBR)', + '29 Run DMC & Jason Nevins - It[br]Pass the toiletpaper', + 'Wachtwoord: geen', + 'WW: zie nzb', + 'BINSEARCH GEEFT SOMS EEN REQUIRES PASSWORD PROTECT, DIT IS NIET JUIST !!', + 'Mijn wachtwoord is hier direct te vinden, je moet het wel zelf overtypen', + 'Bringing History to Life - Greatest Battles of WW II, Sept 2026', + 'WW II', + 'WW Disc 1 = 5YIMKDaoTaKe8WIWEKUF', + 'U kunt eigenaars- en gebruikerswachtwoorden aan uw pdf toevoegen met behulp van 256-bits encryptie.', + 'APP MISC - SHAREit v6.27.49 ww 4062749 arm64 v8a.apk', + 'Het ww staat op het plaatje, type het EXACT zo in.', + 'Wachtwoord:[br]Deze download heeft er geen', + ]) { + assert.isNull(passwordFromSpot('T', description), description) + } + }) +})