Spotnet import, header backfill, staging and counter grouping, faster search
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
bcb9be5da9
commit
be8a3d4580
97 files changed
+4454
-268
No files matched your search
+270
-53
@@ -1,28 +1,55 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import NzbImportService from '#services/NzbImportService'
|
||||
import { type OverviewHeader } from '#services/IngestService'
|
||||
import IngestService, { type OverviewHeader } from '#services/IngestService'
|
||||
import { parseNzb } from '#lib/nzb/index'
|
||||
import {
|
||||
moderationTarget,
|
||||
passwordFromSpot,
|
||||
spotterId,
|
||||
decodeSpotBinary,
|
||||
decodeSpotText,
|
||||
parseSpotArticleHeaders,
|
||||
parseSpotHeader,
|
||||
parseSpotXml,
|
||||
verifySpotHeader,
|
||||
type SpotHeader,
|
||||
type SpotXml,
|
||||
} from '#lib/spotnet/index'
|
||||
|
||||
/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
|
||||
const MAX_NZB_SEGMENTS = 20
|
||||
/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
|
||||
const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
|
||||
/** Rows per INSERT: 14 columns each, under Postgres' 65,535 bind parameters. */
|
||||
const INSERT_CHUNK = 2000
|
||||
/** Spot images larger than this many articles aren't fetched. */
|
||||
const MAX_IMAGE_SEGMENTS = 10
|
||||
/** A claimed spot whose worker died is taken again after this long. */
|
||||
const RECLAIM_AFTER = '1 hour'
|
||||
|
||||
/**
|
||||
* A spot's title: from its XML, which is decoded properly (the Subject of a
|
||||
* raw header is read as ISO-8859-1), else from the header.
|
||||
*/
|
||||
function spotTitle(head: { header: SpotHeader; xml: SpotXml | null }): string {
|
||||
return head.xml?.title || head.header.title
|
||||
}
|
||||
|
||||
/**
|
||||
* A placeholder NZB (e.g. Dummy.rar + Dummy.nzb, 2.6 KB, shared by many
|
||||
* spots): less than 1% of the size the spot announces (the real NZB is
|
||||
* handed out elsewhere), or under 16 KB in all (text-only spots: requests,
|
||||
* announcements). Its files are not imported, and the spot is hidden.
|
||||
*/
|
||||
export function isDummyNzb(nzb: { files: { segments: { bytes: number }[] }[] }, spotSize: number): boolean {
|
||||
const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
|
||||
return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
|
||||
}
|
||||
|
||||
export function isSpotGroup(name: string): boolean {
|
||||
return indexerConfig.spotGroups.includes(name)
|
||||
}
|
||||
@@ -55,74 +82,115 @@ class SpotService {
|
||||
posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
|
||||
key_id: spot.keyId,
|
||||
spotter_id: spotterId(spot.selfSignedPubKey) || null,
|
||||
// From the overview; titles with accents may fail here (UTF-8 decoding)
|
||||
// and are verified again on their raw header by worker:spots.
|
||||
verified: verifySpotHeader(spot),
|
||||
created_at: now,
|
||||
})
|
||||
}
|
||||
if (rows.length === 0) return 0
|
||||
await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore()
|
||||
for (let i = 0; i < rows.length; i += INSERT_CHUNK) {
|
||||
await db.knexQuery().table('spots').insert(rows.slice(i, i + INSERT_CHUNK)).onConflict('message_id').ignore()
|
||||
}
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */
|
||||
/**
|
||||
* Takes up to `limit` pending spots, marked `processing`: recent ones
|
||||
* (to be imported) first, then the history in posting order, so live spots
|
||||
* don't wait behind a backfill and moderation follows its targets.
|
||||
*/
|
||||
async claim(limit: number): Promise<any[]> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE spots SET status = 'processing', processed_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots
|
||||
WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
|
||||
ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[limit]
|
||||
)
|
||||
const claim = async (filter: string, bindings: unknown[], n: number) =>
|
||||
(
|
||||
await db.rawQuery(
|
||||
`UPDATE spots SET status = 'processing', processed_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots WHERE ${filter}
|
||||
ORDER BY article_number LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[...bindings, n]
|
||||
)
|
||||
).rows as any[]
|
||||
const rows = await claim(`status = 'pending' AND posted_at >= ?`, [this.importCutoff()], limit)
|
||||
if (rows.length < limit) rows.push(...(await claim(`status = 'pending'`, [], limit - rows.length)))
|
||||
if (rows.length < limit) {
|
||||
rows.push(...(await claim(`status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}'`, [], limit - rows.length)))
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
/** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */
|
||||
/** Spots posted before this are listed, not imported: their NZB is read on first download. */
|
||||
importCutoff(): Date {
|
||||
return new Date(Date.now() - indexerConfig.spotImportDays * 24 * 60 * 60 * 1000)
|
||||
}
|
||||
|
||||
/** Old, verified spots that aren't moderation: listed in bulk, without reading anything more. */
|
||||
isListable(spot: any): boolean {
|
||||
return (
|
||||
spot.verified === true &&
|
||||
spot.posted_at !== null &&
|
||||
new Date(spot.posted_at) < this.importCutoff() &&
|
||||
!moderationTarget({ keyId: spot.key_id, title: spot.title })
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the listing releases (source `spot`, no files) for these spots.
|
||||
* The release key is the one an import of the spot's NZB uses, so the
|
||||
* import on first download fills this release.
|
||||
*/
|
||||
async list(spots: any[]): Promise<number> {
|
||||
if (spots.length === 0) return 0
|
||||
const keyOf = (spot: any) => createHash('sha1').update(`nzb\n${spot.message_id}`).digest('hex')
|
||||
const ids = await IngestService.upsertReleases(
|
||||
db.connection().getWriteClient(),
|
||||
spots.map((spot) => ({
|
||||
release: { key: keyOf(spot), name: spot.title, source: 'spot' as const },
|
||||
poster: spot.poster,
|
||||
groups: [spot.group_name],
|
||||
postedAt: spot.posted_at ? new Date(spot.posted_at) : undefined,
|
||||
}))
|
||||
)
|
||||
const rows = spots.map((spot) => ({ message_id: spot.message_id, release_id: ids.get(keyOf(spot)), size: spot.size }))
|
||||
await db.rawQuery(
|
||||
`UPDATE releases r SET size = v.size FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, size bigint)
|
||||
WHERE r.id = v.release_id AND r.file_count = 0 AND v.size IS NOT NULL`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE spots s SET status = 'listed', release_id = v.release_id, processed_at = now()
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, release_id bigint)
|
||||
WHERE s.message_id = v.message_id`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies the spot on its raw header (the signature covers ISO-8859-1
|
||||
* bytes, which the overview, read as UTF-8, may have mangled), then
|
||||
* applies moderation, lists an old spot or imports a recent one's NZB.
|
||||
* Returns the new status.
|
||||
*/
|
||||
async process(spot: any): Promise<string> {
|
||||
const update: Record<string, unknown> = { processed_at: new Date(), error: null }
|
||||
try {
|
||||
const raw = await this.fetch(`HEAD <${spot.message_id}>`)
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
|
||||
// The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled.
|
||||
const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date)
|
||||
if (!header || !verifySpotHeader(header)) {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) {
|
||||
update.status = 'unverified'
|
||||
} else if (moderationTarget(header)) {
|
||||
} else if (moderationTarget(head.header)) {
|
||||
update.status = 'moderation'
|
||||
await this.moderate(header, moderationTarget(header)!)
|
||||
await this.moderate(head.header, moderationTarget(head.header)!)
|
||||
} else {
|
||||
const parsed = parseSpotXml(xml)
|
||||
update.title = header.title
|
||||
update.poster = header.poster
|
||||
update.spotter_id = spotterId(header.selfSignedPubKey) || null
|
||||
if (parsed?.website) update.website = parsed.website
|
||||
const segments = parsed?.nzbSegments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) {
|
||||
update.status = 'no-nzb'
|
||||
update.title = spotTitle(head)
|
||||
update.poster = head.header.poster
|
||||
update.spotter_id = spotterId(head.header.selfSignedPubKey) || null
|
||||
Object.assign(update, this.details(head.xml))
|
||||
if (new Date(head.header.stamp * 1000) < this.importCutoff()) {
|
||||
await this.list([{ ...spot, title: spotTitle(head), poster: head.header.poster }])
|
||||
update.status = 'listed'
|
||||
} else {
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
const nzbXml = decodeSpotBinary(bodies).toString('utf8')
|
||||
const nzb = parseNzb(nzbXml)
|
||||
if (!nzb || nzb.files.length === 0) {
|
||||
update.status = 'no-nzb'
|
||||
} else {
|
||||
const groups = [...new Set(nzb.files.flatMap((f) => f.groups))]
|
||||
const result = await NzbImportService.import(
|
||||
{
|
||||
id: null,
|
||||
filename: `${header.title}.nzb`,
|
||||
poster: header.poster,
|
||||
groups,
|
||||
date: header.stamp * 1000,
|
||||
message_ids: { '1': { id: spot.message_id } },
|
||||
},
|
||||
nzbXml,
|
||||
{ name: header.title, source: 'spot', moveContainer: false }
|
||||
)
|
||||
update.status = result ? 'imported' : 'no-nzb'
|
||||
if (result) update.release_id = result.releaseId
|
||||
}
|
||||
Object.assign(update, await this.importNzb(spot.message_id, head))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
@@ -131,9 +199,149 @@ class SpotService {
|
||||
logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
if (update.status === 'imported' || update.status === 'listed') await this.applyPassword(spot.message_id)
|
||||
return update.status as string
|
||||
}
|
||||
|
||||
/**
|
||||
* The NZB of a listing, on first download: reads and imports it. Returns
|
||||
* the release that now holds its files (the listing's own, unless the NZB
|
||||
* joined files already known elsewhere), or null.
|
||||
*/
|
||||
async importListing(releaseId: number): Promise<number | null> {
|
||||
const spot = await db.from('spots').where('release_id', releaseId).where('status', 'listed').first()
|
||||
if (!spot) return null
|
||||
return this.reimport(spot)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a spot's NZB again and imports it, so its files end up in the
|
||||
* spot's own release under the spot's title. Returns that release, or null.
|
||||
*/
|
||||
async reimport(spot: any): Promise<number | null> {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) return null
|
||||
const result = await this.importNzb(spot.message_id, head)
|
||||
await db
|
||||
.from('spots')
|
||||
.where('message_id', spot.message_id)
|
||||
.update({ ...result, ...this.details(head.xml), processed_at: new Date() })
|
||||
await this.applyPassword(spot.message_id)
|
||||
return (result.release_id as number | undefined) ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* A password the spotter wrote in the spot's title or description (not in
|
||||
* the NZB) goes on its release, unless the release has one already.
|
||||
*/
|
||||
async applyPassword(messageId: string): Promise<string | null> {
|
||||
const spot = await db.from('spots').where('message_id', messageId).whereNotNull('release_id').first()
|
||||
const password = spot ? passwordFromSpot(spot.title, spot.description) : null
|
||||
if (!password) return null
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', spot.release_id)
|
||||
.whereNull('password')
|
||||
.update({ password, password_source: 'spot' })
|
||||
return password
|
||||
}
|
||||
|
||||
/** Columns for what the spot's XML adds to its overview: description, image, website. */
|
||||
private details(xml: SpotXml | null): Record<string, unknown> {
|
||||
return {
|
||||
description: xml?.description || null,
|
||||
image: xml?.image ? JSON.stringify(xml.image) : null,
|
||||
...(xml?.website ? { website: xml.website } : {}),
|
||||
details_at: new Date(),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A spot row with its description and image, reading its full header
|
||||
* first when that hasn't happened yet (old spots are listed from their
|
||||
* overview only). Returns the row as it is when Usenet can't be reached.
|
||||
*/
|
||||
async withDetails(spot: any): Promise<any> {
|
||||
if (spot.details_at) return spot
|
||||
try {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
const update = head ? this.details(head.xml) : { details_at: new Date() }
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
await this.applyPassword(spot.message_id)
|
||||
return { ...spot, ...update, image: head?.xml?.image ?? null }
|
||||
} catch (error: any) {
|
||||
logger.warn(`Spot ${spot.message_id} details: ${error.message}`)
|
||||
// The article is gone (430): nothing to read later either.
|
||||
if (/^4\d\d/.test(String(error.message))) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
}
|
||||
return spot
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the details of up to `limit` imported spots that don't have them
|
||||
* (imported before details were kept), newest first. Returns how many.
|
||||
* Listed spots are left to their first view: there are millions.
|
||||
* Returns how many were read (0 when Usenet can't be reached).
|
||||
*/
|
||||
async fillDetails(limit: number): Promise<number> {
|
||||
const spots = await db
|
||||
.from('spots')
|
||||
.where('status', 'imported')
|
||||
.whereNull('details_at')
|
||||
.orderBy('posted_at', 'desc')
|
||||
.limit(limit)
|
||||
let read = 0
|
||||
for (const spot of spots) if ((await this.withDetails(spot)).details_at) read++
|
||||
return read
|
||||
}
|
||||
|
||||
/** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
|
||||
async image(spot: any): Promise<Buffer | null> {
|
||||
const segments: string[] = spot.image?.segments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_IMAGE_SEGMENTS) return null
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
return decodeSpotBinary(bodies, false)
|
||||
}
|
||||
|
||||
/** The spot's raw header, verified; null when it doesn't verify. */
|
||||
private async readHead(messageId: string): Promise<{ header: SpotHeader; xml: SpotXml | null } | null> {
|
||||
const raw = await this.fetch(`HEAD <${messageId}>`)
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
|
||||
const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
|
||||
if (!header || !verifySpotHeader(header)) return null
|
||||
return { header, xml: parseSpotXml(decodeSpotText(xml)) }
|
||||
}
|
||||
|
||||
/** Downloads the spot's compressed NZB and imports it as a release (D26). */
|
||||
private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
|
||||
const segments = head.xml?.nzbSegments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return { status: 'no-nzb' }
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
const nzbXml = decodeSpotBinary(bodies).toString('utf8')
|
||||
const nzb = parseNzb(nzbXml)
|
||||
if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
|
||||
const { header } = head
|
||||
if (isDummyNzb(nzb, header.size)) return { status: 'dummy' }
|
||||
const title = spotTitle(head)
|
||||
const result = await NzbImportService.import(
|
||||
{
|
||||
id: null,
|
||||
filename: `${title}.nzb`,
|
||||
poster: header.poster,
|
||||
groups: [...new Set(nzb.files.flatMap((f) => f.groups))],
|
||||
date: header.stamp * 1000,
|
||||
first_message_id: messageId,
|
||||
},
|
||||
nzbXml,
|
||||
{ name: title, source: 'spot', moveContainer: false, trusted: true }
|
||||
)
|
||||
return result ? { status: 'imported', release_id: result.releaseId } : { status: 'no-nzb' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a delete/dispose/remove: the target spot is marked `removed`
|
||||
* (its release is then hidden from search). As in Spotweb, a personal
|
||||
@@ -150,6 +358,15 @@ class SpotService {
|
||||
}
|
||||
await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
|
||||
logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
|
||||
// Spots that joined its release (a correction, a repost) take the files over now.
|
||||
if (target.release_id) {
|
||||
const sharing = await db
|
||||
.from('spots')
|
||||
.where('release_id', target.release_id)
|
||||
.where('status', 'imported')
|
||||
.whereNot('message_id', targetId)
|
||||
for (const spot of sharing) await this.reimport(spot)
|
||||
}
|
||||
}
|
||||
|
||||
/** One HEAD or BODY, as raw bytes. */
|
||||
@@ -157,7 +374,7 @@ class SpotService {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
const { data } = await conn._artcmd(command)
|
||||
const { data } = await NntpService.timed<any>(conn._artcmd(command), command)
|
||||
return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
|
||||
} catch (error: any) {
|
||||
// 430 (no such article) is an answer, not a broken connection.
|
||||
|
||||
Reference in new issue
Block a user