Spotnet import, header backfill, staging and counter grouping, faster search
Spotnet (CONCEPT D29): - Index free.pt fully and import spots with their descriptions, categories and NZBs; a spot renames its release, since Spotnet names are more trustworthy than names derived from subjects. - Find passwords in spot descriptions too, and tell "needs a password we have" from "needs a password we don't have". Content inspection: - List the contents of 7z (.7z.001) archives, SFV sets, nicovideo uploads and other archive-like files; mark yEncBin .bin containers as encrypted. - Take real names from yEnc headers for randomly named files, and merge uploads that were split across posts. Storage (D31, D32): - Store segment lists zstd-compressed behind the FileStore accessor. - group:backfill indexes older headers at low priority and pauses when the NVMe or HDD part of the database gets too large. Grouping (D35, D36): - Lone header segments wait in staged_segments until a second segment of their file arrives, which drops the per-article junk binaries. - Counter grouping joins random-subject uploads by poster, [i/M] counter, part count and day. Search and maintenance (D33, D34): - A trigram-indexed search_text column, smaller indexes, estimated stats. - Passes that clean up nonsense names; weekly index maintenance; GC budgets per kind. - Web UI: grey out stale results while a search runs and cancel superseded requests. Count only up to 1,000 matches for the page, which made filter clicks take seconds (Newznab keeps the 10,000 cap). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed
1 parent
bcb9be5da9
commit
be8a3d4580
97 files changed
+4454
-268
No files matched your search
@@ -7,7 +7,7 @@ import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
|
||||
import { fileStem, rarReleaseIdentity } from '#lib/release/index'
|
||||
import { fileStem, isGenericName, rarReleaseIdentity } from '#lib/release/index'
|
||||
|
||||
/** Stored per file and release; a listing of thousands of files is not useful. */
|
||||
const MAX_ENTRIES = 500
|
||||
@@ -59,7 +59,9 @@ class ArchiveService {
|
||||
update.encrypted = true
|
||||
}
|
||||
const better = releaseNameFromContents(info)
|
||||
const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined
|
||||
// A spot's title is kept (CONCEPT.md D29).
|
||||
const renamed =
|
||||
release && better && release.source !== 'spot' && (looksRandom(release.name) || release.source === 'post') ? better : undefined
|
||||
if (renamed) update.name = renamed
|
||||
if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
|
||||
|
||||
@@ -138,7 +140,7 @@ export async function renameFromStoredContents(): Promise<number> {
|
||||
.select('id', 'name', 'source', 'contents')
|
||||
let renamed = 0
|
||||
for (const release of releases) {
|
||||
if (!looksRandom(release.name) && release.source !== 'post') continue
|
||||
if (release.source === 'spot' || (!looksRandom(release.name) && release.source !== 'post')) continue
|
||||
const better = releaseNameFromContents({ entries: release.contents })
|
||||
if (!better || better === release.name) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
@@ -147,11 +149,27 @@ export async function renameFromStoredContents(): Promise<number> {
|
||||
return renamed
|
||||
}
|
||||
|
||||
/** Obfuscated names: one long token of mixed letters and digits, no separators. */
|
||||
/**
|
||||
* Obfuscated names: one long token of mixed letters and digits, no
|
||||
* separators, and the other generated shapes seen on Usenet (LOG.md
|
||||
* 2026-10-01 (3)). Kept in step with SQL's is_readable_name().
|
||||
*/
|
||||
export function looksRandom(name: string): boolean {
|
||||
// An unknown short extension (e.g. ".bin") says nothing about the name.
|
||||
name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||||
// VIDEO_TS, Sample, CD1: real words, but no name.
|
||||
if (isGenericName(name)) return true
|
||||
if (/^[0-9a-f]{16,}$/i.test(name)) return true
|
||||
// 6327d9fa2d6acf3e9a61fb053359d5e7.1790574045 (hex and a Unix time)
|
||||
if (/^[0-9a-f]{16,}\.\d{9,}$/i.test(name)) return true
|
||||
// UUIDs
|
||||
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(name)) return true
|
||||
// 1650777167 (yEncBin), "83111763 n" (rclone uploads)
|
||||
if (/^\d{6,}( n)?$/.test(name)) return true
|
||||
// prvjr7hapd0i1ny14pz4, and rclone's long base32 names
|
||||
if (/^[a-z0-9]{12,}$/.test(name) && (name.match(/\d/g)?.length ?? 0) >= 2 && (name.match(/[a-z]/g)?.length ?? 0) >= 4) return true
|
||||
// wHJ7s71sAGE: short, but capitals all over the token, not only at word starts
|
||||
if (/^[A-Za-z0-9]{8,15}$/.test(name) && /\d/.test(name) && /[a-z]/.test(name) && (name.slice(1).match(/[A-Z]/g)?.length ?? 0) >= 3) return true
|
||||
if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
|
||||
// Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
|
||||
return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import FetchRangeService from '#services/FetchRangeService'
|
||||
import { articleAtDate } from '#lib/nntp/date_search'
|
||||
|
||||
/** Articles XOVERed to find a posting date near an article number. */
|
||||
const PROBE = 200n
|
||||
|
||||
/**
|
||||
* Header backfill (CONCEPT.md D32). `start` gives a group a cursor from
|
||||
* where live indexing began back to the article posted N days ago; `tick`
|
||||
* (every scheduler run) queues low-priority, header-only fetch ranges from
|
||||
* that cursor downwards, newest first, while few are waiting and the
|
||||
* database stays within its size limits.
|
||||
*/
|
||||
class BackfillService {
|
||||
/** Sets the group's backfill to reach back `days` days. Returns its range. */
|
||||
async start(groupName: string, days: number): Promise<{ from: bigint; to: bigint } | null> {
|
||||
const group = await db.from('groups').where('name', groupName).first()
|
||||
if (!group) throw new Error(`Unknown group ${groupName}`)
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
const { first, last } = await NntpService.timed(NntpService.selectGroup(conn, groupName), `GROUP ${groupName}`)
|
||||
const dateOf = async (n: bigint) => {
|
||||
const { overviews } = await NntpService.timed<any>(conn.xover(n, n + PROBE), `XOVER ${groupName} ${n}`)
|
||||
const header = overviews.find(([number, h]: [number | null, any]) => number !== null && h?.date)?.[1]
|
||||
const date = header ? Date.parse(header.date) : Number.NaN
|
||||
return Number.isNaN(date) ? null : date
|
||||
}
|
||||
const until = await articleAtDate(first, last, Date.now() - days * 24 * 60 * 60 * 1000, dateOf)
|
||||
// Live indexing began a little before the group was added: continue from there.
|
||||
const began = await articleAtDate(first, last, new Date(group.created_at).getTime(), dateOf)
|
||||
const next = group.backfill_next_id !== null && BigInt(group.backfill_next_id) < began ? BigInt(group.backfill_next_id) : began
|
||||
await db.from('groups').where('id', group.id).update({ backfill_until_id: until.toString(), backfill_next_id: next.toString() })
|
||||
return next > until ? { from: until, to: next } : null
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
|
||||
/** Stops a group's backfill (queued ranges still run). */
|
||||
async stop(groupName: string) {
|
||||
await db.from('groups').where('name', groupName).update({ backfill_until_id: null, backfill_next_id: null })
|
||||
}
|
||||
|
||||
/** Queues the next backfill ranges. Returns how many. */
|
||||
async tick(): Promise<number> {
|
||||
const { maxQueued, maxDefaultGb, maxHddGb } = indexerConfig.backfill
|
||||
const groups = await db
|
||||
.from('groups')
|
||||
.where('active', true)
|
||||
.whereNotNull('backfill_until_id')
|
||||
.whereRaw('backfill_next_id > backfill_until_id')
|
||||
.orderBy('id')
|
||||
if (groups.length === 0) return 0
|
||||
|
||||
const room = await this.room(maxDefaultGb, maxHddGb)
|
||||
if (room !== true) {
|
||||
logger.warn(`Backfill paused: ${room}.`)
|
||||
return 0
|
||||
}
|
||||
const [{ count }] = await db.from('fetch_ranges').where('backfill', true).where('status', 'queued').count('* as count')
|
||||
let free = maxQueued - Number(count)
|
||||
let queued = 0
|
||||
// Round-robin over the groups, one range each per pass.
|
||||
while (free > 0 && groups.length > 0) {
|
||||
for (const group of [...groups]) {
|
||||
if (free <= 0) break
|
||||
const next = BigInt(group.backfill_next_id)
|
||||
const until = BigInt(group.backfill_until_id)
|
||||
const start = next - indexerConfig.batchSize + 1n > until ? next - indexerConfig.batchSize + 1n : until + 1n
|
||||
await FetchRangeService.queue(group, start, next, { backfill: true })
|
||||
group.backfill_next_id = (start - 1n).toString()
|
||||
await db.from('groups').where('id', group.id).update({ backfill_next_id: group.backfill_next_id })
|
||||
free--
|
||||
queued++
|
||||
if (start - 1n <= until) {
|
||||
groups.splice(groups.indexOf(group), 1)
|
||||
logger.info(`${group.name}: backfill fully queued.`)
|
||||
}
|
||||
}
|
||||
}
|
||||
return queued
|
||||
}
|
||||
|
||||
/** True when both parts of the database are under their limits, else why not. */
|
||||
private async room(maxDefaultGb: number, maxHddGb: number): Promise<true | string> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT spcname, pg_tablespace_size(oid) AS bytes FROM pg_tablespace WHERE spcname IN ('pg_default', 'hdd')`
|
||||
)
|
||||
const gb = (name: string) => Number(rows.find((r: any) => r.spcname === name)?.bytes ?? 0) / 1024 ** 3
|
||||
if (gb('pg_default') > maxDefaultGb) return `database on the NVMe is ${gb('pg_default').toFixed(1)} GB (limit ${maxDefaultGb})`
|
||||
if (gb('hdd') > maxHddGb) return `files table on the HDD is ${gb('hdd').toFixed(1)} GB (limit ${maxHddGb})`
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
export default new BackfillService()
|
||||
+105
-12
@@ -6,10 +6,25 @@ import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#
|
||||
import NameService from '#services/NameService'
|
||||
import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
|
||||
import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
|
||||
import { sniffContentType, type ContentType } from '#lib/content/index'
|
||||
import {
|
||||
isArchiveStart,
|
||||
NICOVIDEO_FILE,
|
||||
NICOVIDEO_RELEASE,
|
||||
nicovideoInfo,
|
||||
nicovideoReleaseName,
|
||||
sniffContentType,
|
||||
zstdPrefix,
|
||||
type ContentType,
|
||||
type NicovideoInfo,
|
||||
} from '#lib/content/index'
|
||||
import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
import { fileStem, releaseIdentity } from '#lib/release/index'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { looksLikeFilename } from '#lib/subject/index'
|
||||
import { sevenZipVolume } from '#lib/sevenzip/index'
|
||||
import SevenZipService from '#services/SevenZipService'
|
||||
import SfvService from '#services/SfvService'
|
||||
import { segmentsOf } from '#services/FileStore'
|
||||
|
||||
const FIRST_CHUNK = 16 * 1024
|
||||
|
||||
@@ -35,6 +50,12 @@ export interface ContentResult {
|
||||
nzb?: NzbImportResult
|
||||
/** NZBs read from inside an archive. */
|
||||
archiveNzbs?: NzbImportResult[]
|
||||
/** Files an SFV's listing moved into its release. */
|
||||
sfvMoved?: number
|
||||
/** A niconico capture's release name from its API response (D34). */
|
||||
nicovideo?: string
|
||||
/** The real name from the yEnc header, for a file posted under a random subject (D36). */
|
||||
yencName?: string
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -48,14 +69,65 @@ class ContentService {
|
||||
/** Random or missing names, and named first RAR volumes (for contents and password flag). */
|
||||
isCandidate(name: string) {
|
||||
if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
|
||||
if (isArchiveStart(name)) return true
|
||||
if (/\.nzb$/i.test(name)) return true
|
||||
// SFVs name the files of their release; niconico captures carry their title (D34).
|
||||
if (/\.sfv$/i.test(name) || NICOVIDEO_FILE.test(name)) return true
|
||||
// Every 7z volume: the set is listed when its last volume is in (see inspect).
|
||||
if (sevenZipVolume(name) !== null) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
}
|
||||
|
||||
/**
|
||||
* Uploads that randomise the subject per article often keep the real file
|
||||
* name in the yEnc header inside the body (`Deep.Strike.S01E16.1080p….part01.rar`;
|
||||
* LOG.md 2026-10-02 (2)). A file with a random name takes that name, and
|
||||
* leaves its weak one-file release for the release its name gives, where
|
||||
* the other files of the upload gather. Returns the name, or undefined.
|
||||
*/
|
||||
private async nameFromYenc(file: any, binary: any, yencName: string): Promise<string | undefined> {
|
||||
const current = String(file.real_name ?? file.filename)
|
||||
if (file.real_name || !looksRandom(fileStem(current))) return undefined
|
||||
if (!looksLikeFilename(yencName) || looksRandom(fileStem(yencName)) || yencName === current) return undefined
|
||||
|
||||
const release = file.release_id ? await db.from('releases').where('id', file.release_id).first() : null
|
||||
const weak = !release || (['stem', 'weak-stem'].includes(release.source) && Number(release.file_count) <= 1)
|
||||
let releaseId = file.release_id
|
||||
if (weak) {
|
||||
const identity = releaseIdentity({ poster: file.poster, subject: '', name: yencName, weak: true })
|
||||
if (identity) {
|
||||
const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
|
||||
{ release: identity, poster: file.poster, groups: file.groups ?? [], postedAt: file.date ? new Date(Number(file.date)) : undefined, weakIdentity: true },
|
||||
])
|
||||
releaseId = ids.get(identity.key) ?? releaseId
|
||||
}
|
||||
}
|
||||
await db.from('files').where('id', file.id).update({ real_name: yencName, release_id: releaseId })
|
||||
if (binary) await db.from('binaries').where('id', binary.id).update({ real_name: yencName, release_id: releaseId })
|
||||
return yencName
|
||||
}
|
||||
|
||||
/**
|
||||
* Names a niconico capture's release after the video, while it still has
|
||||
* the archiver's `<time>_nicovideo_jp_watch_sm<id>` name; its tags join
|
||||
* the search text.
|
||||
*/
|
||||
private async nameNicovideo(file: any, info: NicovideoInfo, id: string): Promise<string | undefined> {
|
||||
if (!file.release_id) return undefined
|
||||
const release = await db.from('releases').where('id', file.release_id).first()
|
||||
if (!release || !NICOVIDEO_RELEASE.test(release.name)) return undefined
|
||||
const name = nicovideoReleaseName(info, id)
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', release.id)
|
||||
.update({ name, search_files: info.tags.join(' ').slice(0, 4000) || null })
|
||||
return name
|
||||
}
|
||||
|
||||
/** Downloads a whole (small) file whose first segment is already decoded. */
|
||||
private async readWhole(first: Buffer, messageIds: Record<string, { id: string; size: number }>): Promise<Buffer | null> {
|
||||
const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b))
|
||||
const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0)
|
||||
private async readWhole(first: Buffer, file: any): Promise<Buffer | null> {
|
||||
const parts = segmentsOf(file)
|
||||
const bytes = parts.reduce((sum, p) => sum + (p.size ?? 0), 0)
|
||||
if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
|
||||
if (parts.length === 1) return first
|
||||
|
||||
@@ -63,7 +135,7 @@ class ContentService {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
for (const [, part] of parts) {
|
||||
for (const part of parts) {
|
||||
const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
|
||||
assembled ??= YencFile.fromSegment(segment)
|
||||
assembled.add(segment)
|
||||
@@ -83,7 +155,7 @@ class ContentService {
|
||||
* uploads by other posters.
|
||||
*/
|
||||
private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
|
||||
const archive = await this.readWhole(first, file.message_ids)
|
||||
const archive = await this.readWhole(first, file)
|
||||
const info = archive && parseRarHeaders(archive)
|
||||
if (!archive || !info) return []
|
||||
const entries = info.entries.filter(isNzbEntry)
|
||||
@@ -104,7 +176,13 @@ class ContentService {
|
||||
const file = await db.from('files').where('id', fileId).first()
|
||||
const binary = await db.from('binaries').where('file_id', fileId).first()
|
||||
if (!file || !this.isCandidate(file.filename)) return null
|
||||
const first = file.message_ids?.['1']
|
||||
// A later 7z volume: list the set from its first volume once all volumes are in.
|
||||
if ((sevenZipVolume(String(file.real_name ?? file.filename)) ?? 0) > 1) {
|
||||
const volumes = await SevenZipService.volumes(file)
|
||||
if (!volumes || volumes[0].archive) return null
|
||||
return this.inspect(Number(volumes[0].id))
|
||||
}
|
||||
const first = segmentsOf(file)[0]
|
||||
if (!first?.id) return null
|
||||
|
||||
const conn = await NntpService.acquire()
|
||||
@@ -142,8 +220,9 @@ class ContentService {
|
||||
}
|
||||
|
||||
const result: ContentResult = { named: 0, contentType }
|
||||
if (contentType === 'nzbe' || contentType === 'rclone') {
|
||||
// An encrypted NZB, or rclone crypt (names and data encrypted): nothing
|
||||
result.yencName = await this.nameFromYenc(file, binary, segment.meta.begin.name)
|
||||
if (contentType === 'nzbe' || contentType === 'rclone' || contentType === 'yencbin') {
|
||||
// An encrypted NZB, rclone crypt (names and data encrypted) or yEncBin's container: nothing
|
||||
// to read, and useless without the key.
|
||||
const releaseId = file.release_id ?? binary?.release_id
|
||||
if (releaseId) {
|
||||
@@ -151,14 +230,28 @@ class ContentService {
|
||||
}
|
||||
return result
|
||||
}
|
||||
const nicovideo = NICOVIDEO_FILE.exec(String(file.real_name ?? file.filename))
|
||||
if (nicovideo) {
|
||||
const info = nicovideoInfo(await zstdPrefix(segment.data))
|
||||
if (info) result.nicovideo = await this.nameNicovideo(file, info, nicovideo[1])
|
||||
return result
|
||||
}
|
||||
if (/\.sfv$/i.test(String(file.real_name ?? file.filename))) {
|
||||
const text = await this.readWhole(segment.data, file)
|
||||
if (text) result.sfvMoved = (await SfvService.attach(file, text.toString('latin1'))) ?? undefined
|
||||
return result
|
||||
}
|
||||
if (contentType === 'nzb') {
|
||||
const xml = await this.readWhole(segment.data, file.message_ids)
|
||||
const xml = await this.readWhole(segment.data, file)
|
||||
if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
|
||||
return result
|
||||
}
|
||||
const set = await Par2Service.read(segment.data, file.message_ids)
|
||||
const set = await Par2Service.read(segment.data, segmentsOf(file))
|
||||
if (set && binary) {
|
||||
result.par2 = await Par2Service.attach(binary, set)
|
||||
} else if (contentType === '7z') {
|
||||
const info = await SevenZipService.list(file, segment.data)
|
||||
if (info) result.archive = await ArchiveService.record(file, binary, info)
|
||||
} else {
|
||||
const info = parseRarHeaders(segment.data)
|
||||
if (info) {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import QueueService from '#services/QueueService'
|
||||
import indexerConfig from '#config/indexer'
|
||||
|
||||
/** A failed range is retried after this long... */
|
||||
const REQUEUE_AFTER_MS = 5 * 60 * 1000
|
||||
@@ -20,21 +21,22 @@ interface Group {
|
||||
* retried instead of silently skipped. See CONCEPT.md (D18).
|
||||
*/
|
||||
class FetchRangeService {
|
||||
async queue(group: Group, startId: bigint, endId: bigint) {
|
||||
/** Queues a range. A backfill range is header-only and waits behind live ones (CONCEPT.md D32). */
|
||||
async queue(group: Group, startId: bigint, endId: bigint, { backfill = false } = {}) {
|
||||
const now = new Date()
|
||||
const [row] = await db
|
||||
.table('fetch_ranges')
|
||||
.insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), created_at: now, updated_at: now })
|
||||
.insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), backfill, created_at: now, updated_at: now })
|
||||
.returning(['id', 'requeues'])
|
||||
await this.addJob(group.name, Number(row.id), 0, startId, endId)
|
||||
await this.addJob(group.name, Number(row.id), 0, startId, endId, backfill)
|
||||
}
|
||||
|
||||
private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint) {
|
||||
private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint, backfill: boolean) {
|
||||
await QueueService.nntpFetchQueue.add(
|
||||
'fetch-headers',
|
||||
{ groupName, startId: startId.toString(), endId: endId.toString(), rangeId },
|
||||
{ groupName, startId: startId.toString(), endId: endId.toString(), rangeId, ...(backfill ? { backfill: true } : {}) },
|
||||
// The job id changes per attempt: BullMQ ignores an add() for an id it still knows.
|
||||
{ jobId: this.jobId(rangeId, requeues) }
|
||||
{ jobId: this.jobId(rangeId, requeues), ...(backfill ? { priority: indexerConfig.backfill.priority } : {}) }
|
||||
)
|
||||
}
|
||||
|
||||
@@ -75,7 +77,7 @@ class FetchRangeService {
|
||||
.from('fetch_ranges')
|
||||
.where('id', range.id)
|
||||
.update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() })
|
||||
await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id))
|
||||
await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id), Boolean(range.backfill))
|
||||
requeued++
|
||||
logger.warn(
|
||||
`Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}`
|
||||
@@ -85,13 +87,14 @@ class FetchRangeService {
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves `groups.last_indexed_id` up to just below the first range that is
|
||||
* not done yet, and forgets done ranges. Returns the new watermark.
|
||||
* Moves `groups.last_indexed_id` up to just below the first live range
|
||||
* that is not done yet (backfill ranges lie below it and don't count), and
|
||||
* forgets done ranges. Returns the new watermark.
|
||||
*/
|
||||
async advanceWatermark(group: Group): Promise<bigint> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce(
|
||||
(SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done'),
|
||||
(SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done' AND NOT backfill),
|
||||
g.last_queued_id))
|
||||
WHERE g.id = ? RETURNING last_indexed_id`,
|
||||
[group.id]
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import type { QueryClientContract } from '@adonisjs/lucid/types/database'
|
||||
import { packSegments, unpackSegments, type Segment } from '#lib/nzb/segments'
|
||||
|
||||
/**
|
||||
* The only code that knows how `files` keeps its segment lists (CONCEPT.md
|
||||
* D31): compressed in `files.segments`, with the first segment's message-id
|
||||
* in the indexed `first_message_id` for lookups. Read them with
|
||||
* `segmentsOf(file)`; write new files with `FileStore.insert`.
|
||||
*/
|
||||
|
||||
/** A new `files` row, with its segments as a list. */
|
||||
export type NewFile = Record<string, unknown> & { segments: Segment[] }
|
||||
|
||||
const unpacked = new WeakMap<object, Segment[]>()
|
||||
|
||||
/** The segments of a `files` row (a row from `db.from('files')`), in order. */
|
||||
export function segmentsOf(file: { segments?: Buffer | Segment[] | null }): Segment[] {
|
||||
const { segments } = file
|
||||
if (!segments) return []
|
||||
if (Array.isArray(segments)) return segments
|
||||
let list = unpacked.get(file)
|
||||
if (!list) {
|
||||
list = unpackSegments(segments)
|
||||
unpacked.set(file, list)
|
||||
}
|
||||
return list
|
||||
}
|
||||
|
||||
/** Stored form of a segment list, for the File model's column. */
|
||||
export function storedSegments(segments: Segment[]) {
|
||||
return packSegments(segments)
|
||||
}
|
||||
|
||||
export function loadedSegments(stored: Buffer | null): Segment[] {
|
||||
return stored ? unpackSegments(stored) : []
|
||||
}
|
||||
|
||||
class FileStore {
|
||||
/**
|
||||
* Inserts files, skipping those whose `file_key` exists. Returns the rows
|
||||
* that were inserted (id and filename).
|
||||
*/
|
||||
async insert(files: NewFile[], client: Pick<QueryClientContract, 'knexQuery'> = db): Promise<{ id: number; filename: string }[]> {
|
||||
if (files.length === 0) return []
|
||||
const rows = files.map(({ segments, ...file }) => {
|
||||
const sorted = [...segments].sort((a, b) => a.number - b.number)
|
||||
return { ...file, segments: packSegments(sorted), first_message_id: sorted[0]?.id ?? null }
|
||||
})
|
||||
const inserted = await client.knexQuery().table('files').insert(rows).onConflict('file_key').ignore().returning(['id', 'filename'])
|
||||
return inserted.map((row: any) => ({ id: Number(row.id), filename: row.filename }))
|
||||
}
|
||||
}
|
||||
|
||||
export default new FileStore()
|
||||
@@ -0,0 +1,85 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/** Only indexes at least this big are worth checking. */
|
||||
const MIN_BYTES = 50 * 1024 * 1024
|
||||
/** Leaf pages less full than this (percent) mean the index is mostly empty space. */
|
||||
const MAX_LEAF_DENSITY = 50
|
||||
|
||||
export interface Rebuilt {
|
||||
index: string
|
||||
density: number
|
||||
before: number
|
||||
after: number
|
||||
}
|
||||
|
||||
/**
|
||||
* GC deletes millions of rows (single-article binaries, copied parts, empty
|
||||
* releases). Vacuum makes their index entries reusable but doesn't compact
|
||||
* the index, so B-trees on the churning tables end up 10–20% full (LOG.md
|
||||
* 2026-10-01). Rebuilding them online brings them back to size (CONCEPT.md
|
||||
* D33). Needs the pgstattuple extension.
|
||||
*/
|
||||
class IndexMaintenance {
|
||||
async rebuildBloated({ dryRun = false } = {}): Promise<Rebuilt[]> {
|
||||
const { rows: extension } = await db.rawQuery(`SELECT 1 FROM pg_extension WHERE extname = 'pgstattuple'`)
|
||||
if (extension.length === 0) {
|
||||
logger.warn('Index maintenance skipped: the pgstattuple extension is not installed.')
|
||||
return []
|
||||
}
|
||||
if (!dryRun) await this.dropLeftovers()
|
||||
|
||||
const { rows: candidates } = await db.rawQuery(
|
||||
`SELECT c.relname AS index, pg_relation_size(c.oid) AS bytes
|
||||
FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_am am ON am.oid = c.relam
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE am.amname = 'btree' AND n.nspname = 'public' AND i.indisvalid AND pg_relation_size(c.oid) >= ?
|
||||
ORDER BY 2 DESC`,
|
||||
[MIN_BYTES]
|
||||
)
|
||||
const rebuilt: Rebuilt[] = []
|
||||
for (const { index, bytes } of candidates) {
|
||||
const { rows } = await db.rawQuery(`SELECT avg_leaf_density FROM pgstatindex(?)`, [index])
|
||||
const density = Number(rows[0].avg_leaf_density)
|
||||
if (density >= MAX_LEAF_DENSITY) continue
|
||||
if (!dryRun && !(await this.reindex(index))) continue
|
||||
const { rows: after } = await db.rawQuery(`SELECT pg_relation_size(?::regclass) AS bytes`, [index])
|
||||
const result = { index, density, before: Number(bytes), after: Number(after[0].bytes) }
|
||||
rebuilt.push(result)
|
||||
logger.info(`Index ${index}: ${density.toFixed(0)}% full, ${mb(result.before)} → ${mb(result.after)}${dryRun ? ' (dry run)' : ''}.`)
|
||||
}
|
||||
return rebuilt
|
||||
}
|
||||
|
||||
/**
|
||||
* REINDEX CONCURRENTLY can lose a deadlock against busy writers (seen on
|
||||
* binaries during a backfill). Its half-built copy is dropped and it is
|
||||
* tried once more; a second failure skips the index until the next run.
|
||||
*/
|
||||
/** Invalid copies an interrupted REINDEX CONCURRENTLY left behind (`<index>_ccnew`, `_ccnew1`, …). */
|
||||
private async dropLeftovers(index = '') {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT c.relname FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid
|
||||
WHERE NOT i.indisvalid AND c.relname LIKE ? || '%\\_ccnew%'`,
|
||||
[index]
|
||||
)
|
||||
for (const { relname } of rows) await db.rawQuery(`DROP INDEX CONCURRENTLY IF EXISTS "${relname}"`)
|
||||
}
|
||||
|
||||
private async reindex(index: string): Promise<boolean> {
|
||||
for (let attempt = 1; attempt <= 2; attempt++) {
|
||||
try {
|
||||
await db.rawQuery(`REINDEX INDEX CONCURRENTLY "${index}"`)
|
||||
return true
|
||||
} catch (error: any) {
|
||||
logger.warn(`Rebuilding index ${index} failed (attempt ${attempt}): ${error.message}`)
|
||||
await this.dropLeftovers(index)
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
const mb = (bytes: number) => `${Math.round(bytes / 1024 / 1024)} MB`
|
||||
|
||||
export default new IndexMaintenance()
|
||||
@@ -112,7 +112,7 @@ export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; r
|
||||
const { par2ReleaseName } = await import('#lib/release/index')
|
||||
const listings = await db.rawQuery(
|
||||
`SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r
|
||||
JOIN release_files rf ON rf.release_id = r.id GROUP BY r.id, r.name`
|
||||
JOIN release_files rf ON rf.release_id = r.id WHERE r.source <> 'spot' GROUP BY r.id, r.name`
|
||||
)
|
||||
let renamed = 0
|
||||
for (const release of listings.rows) {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import nntpConfig from '#config/nntp'
|
||||
import { NNTP } from "nntp-js";
|
||||
import { NNTP } from "#lib/nntp/nntp_js";
|
||||
import { parseGroupResponse, type GroupInfo } from '#lib/nntp/group'
|
||||
|
||||
class NntpService {
|
||||
@@ -94,6 +94,20 @@ class NntpService {
|
||||
return parseGroupResponse(resp)
|
||||
}
|
||||
|
||||
/**
|
||||
* Rejects when `promise` hasn't settled within `ms`. nntp-js has no
|
||||
* timeouts: a connection the server stops answering (still ESTABLISHED,
|
||||
* nothing in flight) leaves its command waiting forever. The caller must
|
||||
* then release the connection as failed, so it's destroyed.
|
||||
*/
|
||||
public timed<T>(promise: Promise<T>, what: string, ms = nntpConfig.commandTimeoutMs): Promise<T> {
|
||||
let timer: NodeJS.Timeout
|
||||
const timeout = new Promise<never>((_, reject) => {
|
||||
timer = setTimeout(() => reject(new Error(`NNTP ${what}: no answer within ${ms / 1000} s`)), ms)
|
||||
})
|
||||
return Promise.race([promise, timeout]).finally(() => clearTimeout(timer))
|
||||
}
|
||||
|
||||
/** Fetches a raw (still dot-stuffed) article body as a Buffer. */
|
||||
public async fetchBody(conn: any, messageId: string): Promise<Buffer> {
|
||||
const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
|
||||
|
||||
@@ -7,6 +7,9 @@ import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
||||
import { parseSubject } from '#lib/subject/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import FileStore from '#services/FileStore'
|
||||
import { isArchiveStart } from '#lib/content/index'
|
||||
|
||||
export interface NzbImportResult {
|
||||
releaseId: number
|
||||
@@ -20,8 +23,29 @@ export interface NzbImportResult {
|
||||
/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
|
||||
export function nzbFileName(subject: string): string {
|
||||
const parsed = parseSubject(subject)
|
||||
const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject
|
||||
return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
|
||||
const quoted = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1]
|
||||
if (quoted) return quoted.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
|
||||
return unquotedFileName(subject)
|
||||
}
|
||||
|
||||
/**
|
||||
* A filename from a subject without quotes, as some spot NZBs have:
|
||||
* `Adventure.part0055.rar - [056/175] (1/1)`, `x.vol007+008.par2 (1/0) (1/0)`,
|
||||
* `06 - Song.flac Artist - Album (2020) (01/35)`. Counters, "yEnc" and a
|
||||
* trailing size are dropped; then the name ends at the first extension
|
||||
* followed by a space.
|
||||
*/
|
||||
export function unquotedFileName(subject: string): string {
|
||||
const cleaned = subject
|
||||
.replace(/\(\d+\/\d+\)|\[\d+\/\d+\]|\byEnc\b/gi, ' ')
|
||||
.replace(/\s\d+\s*$/, ' ')
|
||||
.replace(/(\s+-)+\s*$/, '')
|
||||
.replace(/^\s*-\s+/, '')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
const end = /\.(?:[A-Za-z][A-Za-z0-9]{1,4}|\d{3})(?=\s)/.exec(cleaned)
|
||||
const name = end ? cleaned.slice(0, end.index + end[0].length) : cleaned
|
||||
return name.replace(/\s+-$/, '').trim() || subject.trim()
|
||||
}
|
||||
|
||||
export interface NzbImportOptions {
|
||||
@@ -36,6 +60,13 @@ export interface NzbImportOptions {
|
||||
name?: string
|
||||
/** Release source for new releases (default `nzb`). */
|
||||
source?: ReleaseIdentity['source']
|
||||
/**
|
||||
* The NZB is authoritative for which files form the release and what it is
|
||||
* called (Spotnet spots): its files get the NZB's own release, named
|
||||
* `name`, also when name rules had put them in another release. Files
|
||||
* another trusted NZB claimed first stay where they are.
|
||||
*/
|
||||
trusted?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -55,21 +86,39 @@ class NzbImportService {
|
||||
firstId: f.segments[0].id,
|
||||
}))
|
||||
|
||||
return db.transaction(async (trx) => {
|
||||
const added: { id: number; filename: string }[] = []
|
||||
const result = await db.transaction(async (trx) => {
|
||||
const known = await trx
|
||||
.from('files')
|
||||
.whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)])
|
||||
.select('id', 'release_id', `message_ids`)
|
||||
const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id))
|
||||
.whereIn('first_message_id', entries.map((e) => e.firstId))
|
||||
.select('id', 'release_id', 'first_message_id')
|
||||
const knownIds = new Set(known.map((f) => f.first_message_id))
|
||||
|
||||
// Join the release the listed files are already in, else the posted NZB's own, else a new one.
|
||||
const counts = new Map<number, number>()
|
||||
for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
|
||||
let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
|
||||
const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
|
||||
const origin = `nzb\n${nzbFile.first_message_id ?? nzbFile.id}`
|
||||
const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
|
||||
|
||||
// Releases of other trusted NZBs: their files are not taken over.
|
||||
let claimed = new Set<number>()
|
||||
if (options.trusted) {
|
||||
// Only while that NZB still stands: a disposed spot's files go to its correction.
|
||||
const holders = await trx
|
||||
.from('releases as r')
|
||||
.whereIn('r.id', [...counts.keys()])
|
||||
.where('r.source', 'spot')
|
||||
.whereNot('r.release_key', key)
|
||||
.whereNotExists((q) => q.from('spots as s').whereRaw('s.release_id = r.id').whereIn('s.status', ['removed', 'dummy']))
|
||||
.select('r.id')
|
||||
claimed = new Set(holders.map((r) => Number(r.id)))
|
||||
// Join another spot's release when it holds most of these files (a repost); else use our own.
|
||||
if (releaseId && !claimed.has(releaseId)) releaseId = undefined
|
||||
if (releaseId) claimed.delete(releaseId)
|
||||
}
|
||||
if (!releaseId) {
|
||||
const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}`
|
||||
const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
|
||||
const ids = await IngestService.upsertReleases(trx.knexClient, [
|
||||
{
|
||||
release: { key, name: title, source: options.source ?? 'nzb' },
|
||||
@@ -84,9 +133,8 @@ class NzbImportService {
|
||||
const now = new Date()
|
||||
const missing = entries.filter((e) => !knownIds.has(e.firstId))
|
||||
for (let i = 0; i < missing.length; i += 500) {
|
||||
await trx
|
||||
.knexClient('files')
|
||||
.insert(
|
||||
added.push(
|
||||
...(await FileStore.insert(
|
||||
missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
|
||||
file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
|
||||
filename: name,
|
||||
@@ -94,22 +142,23 @@ class NzbImportService {
|
||||
date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
|
||||
parts: file.segments.length,
|
||||
size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
|
||||
message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))),
|
||||
segments: file.segments.map((s) => ({ number: s.number, id: s.id, size: s.bytes })),
|
||||
groups: JSON.stringify(file.groups),
|
||||
release_id: releaseId,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}))
|
||||
)
|
||||
.onConflict('file_key')
|
||||
.ignore()
|
||||
})),
|
||||
trx
|
||||
))
|
||||
)
|
||||
}
|
||||
// The poster's NZB says which files form the post: known files join this
|
||||
// release, also when name rules had split them over several.
|
||||
const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))]
|
||||
const moving = known.filter((f) => Number(f.release_id) !== releaseId && !claimed.has(Number(f.release_id)))
|
||||
const previous = [...new Set(moving.map((f) => f.release_id).filter(Boolean))]
|
||||
await trx
|
||||
.from('files')
|
||||
.whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id))
|
||||
.whereIn('id', moving.map((f) => f.id))
|
||||
.update({ release_id: releaseId })
|
||||
if (previous.length) {
|
||||
await trx.rawQuery(
|
||||
@@ -130,9 +179,15 @@ class NzbImportService {
|
||||
|
||||
const release = await trx.from('releases').where('id', releaseId).first()
|
||||
const update: Record<string, unknown> = {}
|
||||
if (nzb.meta.password) update.password = nzb.meta.password
|
||||
if (nzb.meta.password) Object.assign(update, { password: nzb.meta.password, password_source: 'nzb' })
|
||||
if (!release.file_total) update.file_total = entries.length
|
||||
if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title
|
||||
if (options.trusted) {
|
||||
// Only its own release: a repost joining another spot's release keeps that one's title.
|
||||
if (release.release_key === key && release.name !== title) update.name = title
|
||||
if (release.release_key === key && options.source && release.source !== options.source) update.source = options.source
|
||||
} else if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) {
|
||||
update.name = title
|
||||
}
|
||||
if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
|
||||
|
||||
const result = {
|
||||
@@ -149,6 +204,14 @@ class NzbImportService {
|
||||
)
|
||||
return result
|
||||
})
|
||||
// Their first segment tells what the archives hold (contents, password); nothing else reads it for files from an NZB.
|
||||
const archives = added.filter((f) => isArchiveStart(f.filename))
|
||||
if (archives.length) {
|
||||
await QueueService.collectionQueue.addBulk(
|
||||
archives.map((f) => ({ name: 'process-collection', data: { fileId: f.id }, opts: { jobId: `file-${f.id}` } }))
|
||||
)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import type { Segment } from '#lib/nzb/index'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NntpService from '#services/NntpService'
|
||||
import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
|
||||
@@ -35,11 +36,9 @@ class Par2Service {
|
||||
* Reads a par2 file whose first segment is already decoded, downloading the
|
||||
* remaining parts if there are any. Returns null when it is not par2.
|
||||
*/
|
||||
async read(first: Buffer, messageIds: Record<string, { id: string }>): Promise<Par2Set | null> {
|
||||
async read(first: Buffer, segments: Segment[]): Promise<Par2Set | null> {
|
||||
if (!isPar2(first)) return null
|
||||
const ids = Object.entries(messageIds)
|
||||
.sort(([a], [b]) => Number(a) - Number(b))
|
||||
.map(([, part]) => part.id)
|
||||
const ids = segments.map((part) => part.id)
|
||||
if (ids.length > MAX_PARTS) return null
|
||||
if (ids.length === 1) return parsePar2(first)
|
||||
|
||||
@@ -143,9 +142,11 @@ class Par2Service {
|
||||
)
|
||||
|
||||
// The set may have joined a release named after one random filename.
|
||||
let releaseName = (await trx.from('releases').where('id', releaseId).first())?.name
|
||||
const release = await trx.from('releases').where('id', releaseId).first()
|
||||
let releaseName = release?.name
|
||||
const better = par2ReleaseName(set.files.map((f) => f.name))
|
||||
if (releaseName && better && isBetterName(releaseName, better)) {
|
||||
// A spot's title is kept (CONCEPT.md D29).
|
||||
if (releaseName && better && release.source !== 'spot' && isBetterName(releaseName, better)) {
|
||||
await trx.from('releases').where('id', releaseId).update({ name: better })
|
||||
releaseName = better
|
||||
}
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
|
||||
/**
|
||||
* Uploads split per poster (CONCEPT.md D35). Some uploaders post every
|
||||
* volume of a RAR set under a different poster; the post-id grouping then
|
||||
* makes one release per volume, all with the same name
|
||||
* (`Masters.of.the.Universe.1987…`: 191 one-file releases of one 199-volume
|
||||
* set). Such releases are one upload when all their files share one stem and
|
||||
* no file name repeats (a repost would repeat them, another upload would
|
||||
* bring another stem); their files move into the oldest of them, and GC
|
||||
* deletes the emptied ones.
|
||||
*/
|
||||
class ReleaseMergeService {
|
||||
async mergeSplitPosts({ dryRun = false } = {}): Promise<{ names: number; releases: number }> {
|
||||
const { rows } = await db.rawQuery(`
|
||||
WITH dup AS (
|
||||
SELECT name FROM releases
|
||||
WHERE file_count > 0 AND source = 'post' AND is_readable_name(name)
|
||||
GROUP BY name HAVING count(*) > 1
|
||||
)
|
||||
SELECT r.name, array_agg(DISTINCT r.id ORDER BY r.id) AS ids
|
||||
FROM dup JOIN releases r ON r.name = dup.name AND r.file_count > 0 AND r.source = 'post'
|
||||
JOIN files f ON f.release_id = r.id
|
||||
GROUP BY r.name
|
||||
HAVING count(DISTINCT name_stem(coalesce(f.real_name, f.filename))) = 1
|
||||
AND count(f.id) = count(DISTINCT coalesce(f.real_name, f.filename))
|
||||
AND max(r.posted_at) - min(r.posted_at) < interval '30 days'`)
|
||||
let releases = 0
|
||||
for (const { name, ids } of rows as { name: string; ids: (string | number)[] }[]) {
|
||||
const [target, ...others] = ids.map(Number)
|
||||
releases += others.length
|
||||
if (dryRun) continue
|
||||
await db.from('files').whereIn('release_id', others).update({ release_id: target })
|
||||
await db.from('binaries').whereIn('release_id', others).update({ release_id: target })
|
||||
logger.info(`Merged ${others.length + 1} releases of one upload split per poster into ${target} "${name}".`)
|
||||
}
|
||||
return { names: rows.length, releases }
|
||||
}
|
||||
}
|
||||
|
||||
export default new ReleaseMergeService()
|
||||
+149
-32
@@ -1,6 +1,20 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { buildNzb } from '#lib/nzb/index'
|
||||
import SpotService from '#services/SpotService'
|
||||
import { segmentsOf } from '#services/FileStore'
|
||||
import { spotCategoryLabels } from '#lib/spotnet/index'
|
||||
|
||||
/**
|
||||
* `known`: we have the password (it's in our NZB, so SABnzbd/NZBGet unpack
|
||||
* without asking); `missing`: the archive is encrypted and we don't have
|
||||
* it; `none`: no password needed as far as we know.
|
||||
*/
|
||||
export type PasswordState = 'known' | 'missing' | 'none'
|
||||
|
||||
export function passwordState(encrypted: boolean | null, hasPassword: boolean): PasswordState {
|
||||
if (hasPassword) return 'known'
|
||||
return encrypted ? 'missing' : 'none'
|
||||
}
|
||||
|
||||
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
||||
|
||||
@@ -10,7 +24,7 @@ export interface SearchParams {
|
||||
readable?: boolean
|
||||
/** Only releases that have all their files. */
|
||||
complete?: boolean
|
||||
/** Hide password-protected releases. */
|
||||
/** Hide releases that need a password we don't have. */
|
||||
hideEncrypted?: boolean
|
||||
source?: string
|
||||
poster?: string
|
||||
@@ -31,6 +45,8 @@ export interface SearchParams {
|
||||
hideContainers?: boolean
|
||||
/** Include incomplete releases that consist of par2 files only (hidden by default). */
|
||||
includePar2Only?: boolean
|
||||
/** Count matches up to this many past the page (default COUNT_LIMIT); `total` is capped there. */
|
||||
countLimit?: number
|
||||
}
|
||||
|
||||
const SORTS: Record<SortOrder, string> = {
|
||||
@@ -58,6 +74,24 @@ function pattern(word: string): string {
|
||||
return `%${escaped.replace(/[._\- ]+/g, '_')}%`
|
||||
}
|
||||
|
||||
/**
|
||||
* As SQL's search_normalize(): lowercase, and dots, dashes, underscores and
|
||||
* spaces as one space. `The.Block.AU` and `the block au` search the same.
|
||||
*/
|
||||
export function searchNormalize(text: string): string {
|
||||
return text.toLowerCase().replace(/[._\s-]+/g, ' ').trim()
|
||||
}
|
||||
|
||||
/** How long the web UI's totals are reused. */
|
||||
const STATS_TTL_MS = 60 * 1000
|
||||
|
||||
/**
|
||||
* Counting stops here: a broad search needn't count every match. Each
|
||||
* counted row is a random heap read, so the web UI counts less (it only
|
||||
* shows "1,000+"); Newznab clients page by this total and keep the default.
|
||||
*/
|
||||
const COUNT_LIMIT = 10_000
|
||||
|
||||
/**
|
||||
* Searching and browsing releases for the web interface (CONCEPT.md D22).
|
||||
*/
|
||||
@@ -66,21 +100,19 @@ class SearchService {
|
||||
const where: string[] = [
|
||||
VISIBLE_SQL,
|
||||
// Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
|
||||
`NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status = 'removed')`,
|
||||
`NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status IN ('removed', 'dummy'))`,
|
||||
]
|
||||
const bindings: unknown[] = []
|
||||
|
||||
const words = (params.q ?? '').trim().split(/\s+/).filter(Boolean).slice(0, 8)
|
||||
// Every word must occur in the release's search text: its name, archive
|
||||
// listing and file names (CONCEPT.md D33), one trigram index for all.
|
||||
const words = (params.q ?? '').trim().split(/\s+/).map(searchNormalize).filter(Boolean).slice(0, 8)
|
||||
for (const word of words) {
|
||||
const p = pattern(word)
|
||||
// Every word must match the release name, a file name, or the archive contents.
|
||||
// One trigram-indexed lookup per table, not a subquery per release.
|
||||
where.push(`(r.name ILIKE ? OR r.contents::text ILIKE ?
|
||||
OR r.id IN (SELECT f.release_id FROM files f WHERE f.filename ILIKE ? OR f.real_name ILIKE ?))`)
|
||||
bindings.push(p, p, p, p)
|
||||
where.push(`r.search_text LIKE ?`)
|
||||
bindings.push(`%${word.replace(/[\\%_]/g, (c) => `\\${c}`)}%`)
|
||||
}
|
||||
if (params.readable) where.push('is_readable_name(r.name)')
|
||||
if (params.hideEncrypted) where.push('r.encrypted IS NOT TRUE')
|
||||
if (params.hideEncrypted) where.push('(r.encrypted IS NOT TRUE OR r.password IS NOT NULL)')
|
||||
if (params.hideContainers) where.push('r.container IS NULL')
|
||||
// par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
|
||||
if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
|
||||
@@ -121,6 +153,7 @@ class SearchService {
|
||||
const offset = params.offset ?? (page - 1) * perPage
|
||||
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
||||
const whereSql = where.join(' AND ')
|
||||
const countLimit = offset + (params.countLimit ?? COUNT_LIMIT)
|
||||
|
||||
const [{ rows }, { rows: countRows }] = await Promise.all([
|
||||
db.rawQuery(
|
||||
@@ -131,19 +164,22 @@ class SearchService {
|
||||
SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
|
||||
jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
|
||||
r.complete, ${PAR2_ONLY_SQL} AS par2_only
|
||||
r.complete, ${PAR2_ONLY_SQL} AS par2_only, r.password IS NOT NULL AS has_password,
|
||||
(SELECT s.description FROM spots s WHERE s.release_id = r.id AND s.description IS NOT NULL LIMIT 1) AS spot_description
|
||||
FROM releases r WHERE ${whereSql}
|
||||
ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
[...bindings, perPage, offset]
|
||||
),
|
||||
db.rawQuery(`SELECT count(*) AS total FROM releases r WHERE ${whereSql}`, bindings),
|
||||
db.rawQuery(`SELECT count(*) AS total FROM (SELECT 1 FROM releases r WHERE ${whereSql} LIMIT ${countLimit + 1}) x`, bindings),
|
||||
])
|
||||
|
||||
return {
|
||||
page,
|
||||
perPage,
|
||||
offset,
|
||||
total: Number(countRows[0].total),
|
||||
total: Math.min(Number(countRows[0].total), countLimit),
|
||||
/** More than `total` match (the count stops at `countLimit`). */
|
||||
totalCapped: Number(countRows[0].total) > countLimit,
|
||||
releases: rows.map((r: any) => ({
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
@@ -163,6 +199,10 @@ class SearchService {
|
||||
category: r.category,
|
||||
container: r.container,
|
||||
par2Only: r.par2_only,
|
||||
listingOnly: r.source === 'spot' && Number(r.file_count) === 0,
|
||||
hasPassword: r.has_password,
|
||||
passwordState: passwordState(r.encrypted, r.has_password),
|
||||
spotDescription: r.spot_description,
|
||||
})),
|
||||
}
|
||||
}
|
||||
@@ -193,14 +233,32 @@ class SearchService {
|
||||
),
|
||||
])
|
||||
|
||||
const spot = release.source === 'spot'
|
||||
? await db.from('spots').where('release_id', id).select('tag', 'poster', 'category', 'website').first()
|
||||
: null
|
||||
const spotRow = await db.from('spots').where('release_id', id).whereNotIn('status', ['removed', 'dummy']).orderBy('posted_at', 'desc').first()
|
||||
const spot = spotRow ? await SpotService.withDetails(spotRow) : null
|
||||
return {
|
||||
id: Number(release.id),
|
||||
name: release.name,
|
||||
source: release.source,
|
||||
spot: spot ? { tag: spot.tag, poster: spot.poster, category: spot.category, website: spot.website } : null,
|
||||
spot: spot
|
||||
? {
|
||||
messageId: spot.message_id,
|
||||
title: spot.title,
|
||||
tag: spot.tag,
|
||||
poster: spot.poster,
|
||||
postedAt: spot.posted_at,
|
||||
website: spot.website,
|
||||
description: spot.description,
|
||||
image: spot.image?.segments
|
||||
? { url: `/api/v1/releases/${id}/image`, width: spot.image.width ?? null, height: spot.image.height ?? null }
|
||||
: spot.image?.url
|
||||
? { external: spot.image.url }
|
||||
: null,
|
||||
categories: spotCategoryLabels(spot.category, spot.subcats),
|
||||
}
|
||||
: null,
|
||||
password: release.password ?? null,
|
||||
passwordSource: release.password_source ?? null,
|
||||
passwordState: passwordState(release.encrypted, release.password !== null),
|
||||
listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
|
||||
poster: release.poster,
|
||||
groups: release.groups,
|
||||
@@ -230,6 +288,22 @@ class SearchService {
|
||||
}
|
||||
}
|
||||
|
||||
/** The image of the spot behind a release: bytes and a media type sniffed from them, or null. */
|
||||
async spotImage(id: number): Promise<{ data: Buffer; type: string } | null> {
|
||||
const spot = await db.from('spots').where('release_id', id).whereNotNull('image').orderBy('posted_at', 'desc').first()
|
||||
if (!spot) return null
|
||||
const data = await SpotService.image(spot)
|
||||
if (!data) return null
|
||||
const type =
|
||||
data[0] === 0xff && data[1] === 0xd8 ? 'image/jpeg'
|
||||
: data.subarray(0, 4).toString('latin1') === '\x89PNG' ? 'image/png'
|
||||
: data.subarray(0, 3).toString('latin1') === 'GIF' ? 'image/gif'
|
||||
: data.subarray(0, 2).toString('latin1') === 'BM' ? 'image/bmp'
|
||||
: data.subarray(8, 12).toString('latin1') === 'WEBP' ? 'image/webp'
|
||||
: null
|
||||
return type ? { data, type } : null
|
||||
}
|
||||
|
||||
async nzb(id: number): Promise<{ name: string; xml: string } | null> {
|
||||
let release = await db.from('releases').where('id', id).first()
|
||||
if (!release) return null
|
||||
@@ -244,39 +318,82 @@ class SearchService {
|
||||
.from('files')
|
||||
.where('release_id', id)
|
||||
.orderByRaw('coalesce(real_name, filename)')
|
||||
.select('filename', 'real_name', 'poster', 'date', 'groups', 'message_ids')
|
||||
.select('filename', 'real_name', 'poster', 'date', 'groups', 'segments')
|
||||
const xml = buildNzb(
|
||||
files.map((f) => ({
|
||||
name: f.real_name ?? f.filename,
|
||||
poster: f.poster,
|
||||
date: Number(f.date),
|
||||
groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
|
||||
segments: f.message_ids,
|
||||
segments: segmentsOf(f),
|
||||
})),
|
||||
{ title: release.name, password: release.password ?? undefined }
|
||||
)
|
||||
return { name: release.name, xml }
|
||||
}
|
||||
|
||||
/**
|
||||
* Totals for the web UI header. They count millions of rows (minutes with
|
||||
* the backfill running), so they are computed at most once per
|
||||
* STATS_TTL_MS and never twice at a time; callers in between share it.
|
||||
*/
|
||||
async stats() {
|
||||
if (!this.statsCache || Date.now() - this.statsCache.at > STATS_TTL_MS) {
|
||||
this.statsRefresh ??= this.computeStats()
|
||||
.then((value) => {
|
||||
this.statsCache = { at: Date.now(), value }
|
||||
})
|
||||
// A failed background refresh keeps the previous totals; only a caller that waits sees the error.
|
||||
.catch((error) => {
|
||||
if (!this.statsCache) throw error
|
||||
})
|
||||
.finally(() => {
|
||||
this.statsRefresh = undefined
|
||||
})
|
||||
// Only the very first caller waits; later ones get the previous totals meanwhile.
|
||||
if (!this.statsCache) await this.statsRefresh
|
||||
}
|
||||
return this.statsCache!.value
|
||||
}
|
||||
|
||||
private statsCache?: { at: number; value: Awaited<ReturnType<SearchService['computeStats']>> }
|
||||
private statsRefresh?: Promise<void>
|
||||
|
||||
/**
|
||||
* Estimates, in milliseconds instead of minutes (CONCEPT.md D33): counts
|
||||
* from Postgres's statistics (visible releases = the entries of an index
|
||||
* that covers exactly them), ratios and sizes from a 0.5% sample of the
|
||||
* releases table. Within a few percent once autovacuum has analysed.
|
||||
*/
|
||||
private async computeStats() {
|
||||
const { rows } = await db.rawQuery(`
|
||||
SELECT (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL}) AS releases,
|
||||
(SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL} AND is_readable_name(name)) AS readable_releases,
|
||||
(SELECT count(*) FROM files) AS files,
|
||||
(SELECT coalesce(sum(size), 0) FROM files) AS size,
|
||||
(SELECT count(*) FROM releases WHERE encrypted) AS encrypted_releases,
|
||||
WITH sample AS (
|
||||
SELECT r.name, r.size, r.file_count, r.encrypted, coalesce(r.source, 'unknown') AS source
|
||||
FROM releases r TABLESAMPLE SYSTEM (0.5) WHERE ${VISIBLE_SQL})
|
||||
SELECT (SELECT reltuples FROM pg_class WHERE relname = 'releases_category_idx') AS releases,
|
||||
(SELECT reltuples FROM pg_class WHERE relname = 'files') AS files,
|
||||
(SELECT count(*) FROM sample) AS sampled,
|
||||
(SELECT count(*) FROM sample WHERE is_readable_name(name)) AS sampled_readable,
|
||||
-- Indexed data only: a spot listing's size is what the spot announces.
|
||||
(SELECT coalesce(sum(size), 0) FROM sample WHERE file_count > 0) AS sampled_size,
|
||||
(SELECT count(*) FROM sample WHERE encrypted) AS sampled_encrypted,
|
||||
(SELECT max(updated_at) FROM groups) AS last_indexed_at,
|
||||
(SELECT json_agg(json_build_object('source', source, 'releases', n) ORDER BY n DESC)
|
||||
FROM (SELECT coalesce(source, 'unknown') AS source, count(*) AS n FROM releases r WHERE ${VISIBLE_SQL} GROUP BY 1) s) AS sources`)
|
||||
(SELECT json_agg(json_build_object('source', source, 'n', n) ORDER BY n DESC)
|
||||
FROM (SELECT source, count(*) AS n FROM sample GROUP BY 1) s) AS sources`)
|
||||
const r = rows[0]
|
||||
const releases = Math.max(0, Number(r.releases))
|
||||
const sampled = Number(r.sampled)
|
||||
// Sample counts scaled to all visible releases.
|
||||
const scale = (n: unknown) => (sampled > 0 ? Math.round((Number(n) / sampled) * releases) : 0)
|
||||
return {
|
||||
releases: Number(r.releases),
|
||||
readableReleases: Number(r.readable_releases),
|
||||
files: Number(r.files),
|
||||
size: Number(r.size),
|
||||
encryptedReleases: Number(r.encrypted_releases),
|
||||
approximate: true,
|
||||
releases,
|
||||
readableReleases: scale(r.sampled_readable),
|
||||
files: Math.max(0, Number(r.files)),
|
||||
size: scale(r.sampled_size),
|
||||
encryptedReleases: scale(r.sampled_encrypted),
|
||||
lastIndexedAt: r.last_indexed_at,
|
||||
sources: r.sources ?? [],
|
||||
sources: (r.sources ?? []).map((s: any) => ({ source: s.source, releases: scale(s.n) })),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,202 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { mkdtemp, open, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NntpService from '#services/NntpService'
|
||||
import { segmentsOf } from '#services/FileStore'
|
||||
import { decodeSegment, type DecodedSegment } from '#lib/yenc/index'
|
||||
import type { RarInfo } from '#lib/rar/index'
|
||||
import {
|
||||
checkHeader,
|
||||
listingStart,
|
||||
parseBsdtarListing,
|
||||
parseSevenZipStart,
|
||||
sevenZipVolume,
|
||||
type SevenZipStart,
|
||||
} from '#lib/sevenzip/index'
|
||||
|
||||
const run = promisify(execFile)
|
||||
|
||||
/** The tail read for a listing: header plus packed header; more is not a header. */
|
||||
const MAX_TAIL_BYTES = 16 * 1024 * 1024
|
||||
|
||||
/**
|
||||
* Lists 7z archives, also split ones (.7z.001, .002, …), from the first
|
||||
* segment of the first volume and the last segment(s) of the last one
|
||||
* (CONCEPT.md D19). Those bytes are put at their place in a sparse file the
|
||||
* size of the whole archive, which bsdtar (libarchive) lists: it only reads
|
||||
* the start and the header at the end.
|
||||
*/
|
||||
class SevenZipService {
|
||||
/**
|
||||
* The volumes of the archive `file` belongs to, in order, when all of them
|
||||
* up to the highest number seen are there; null otherwise. An unsplit (or
|
||||
* unnamed) archive is its own only volume.
|
||||
*/
|
||||
async volumes(file: any): Promise<any[] | null> {
|
||||
const name = String(file.real_name ?? file.filename)
|
||||
const volume = sevenZipVolume(name)
|
||||
if (!volume) return [file]
|
||||
if (!file.release_id) return null
|
||||
const base = name.replace(/\.\d+$/, '').toLowerCase()
|
||||
const rows = await db
|
||||
.from('files')
|
||||
.where('release_id', file.release_id)
|
||||
.whereRaw(`lower(coalesce(real_name, filename)) LIKE ? ESCAPE '\\'`, [`${base.replace(/[\\%_]/g, '\\$&')}.%`])
|
||||
const byNumber = new Map<number, any>()
|
||||
for (const row of rows) {
|
||||
const n = sevenZipVolume(String(row.real_name ?? row.filename))
|
||||
if (n && !byNumber.has(n)) byNumber.set(n, row)
|
||||
}
|
||||
const max = Math.max(0, ...byNumber.keys())
|
||||
const ordered = Array.from({ length: max }, (_, i) => byNumber.get(i + 1))
|
||||
return max > 0 && ordered.every(Boolean) ? ordered : null
|
||||
}
|
||||
|
||||
/**
|
||||
* Lists the archive whose first volume starts with `firstData` (a decoded
|
||||
* first segment). Null when it isn't a 7z start, when volumes are missing,
|
||||
* or when the tail doesn't hold the header its start points to.
|
||||
*/
|
||||
async list(firstFile: any, firstData: Buffer): Promise<RarInfo | null> {
|
||||
const start = parseSevenZipStart(firstData)
|
||||
if (!start) return null
|
||||
const volumes = await this.volumes(firstFile)
|
||||
if (!volumes) return null
|
||||
if (start.headerSize === 0) return this.info([], false, false, volumes.length > 1)
|
||||
|
||||
const tail = await this.readTail(volumes[volumes.length - 1], start)
|
||||
if (!tail) return null
|
||||
|
||||
const dir = await mkdtemp(path.join(tmpdir(), '7z-'))
|
||||
try {
|
||||
const archive = path.join(dir, 'archive.7z')
|
||||
const handle = await open(archive, 'w')
|
||||
try {
|
||||
await handle.write(firstData, 0, Math.min(firstData.length, start.archiveSize), 0)
|
||||
await handle.write(tail.data, 0, tail.data.length, tail.from)
|
||||
await handle.truncate(start.archiveSize)
|
||||
} finally {
|
||||
await handle.close()
|
||||
}
|
||||
return await this.bsdtar(archive, volumes.length > 1)
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/** Lists the sparse archive; a trial extraction of one entry tells whether its data is encrypted. */
|
||||
private async bsdtar(archive: string, split: boolean): Promise<RarInfo | null> {
|
||||
let stdout = ''
|
||||
try {
|
||||
stdout = (await run('bsdtar', ['-tvf', archive], { timeout: 60_000, maxBuffer: 32 * 1024 * 1024 })).stdout
|
||||
} catch (error: any) {
|
||||
if (/header is encrypted/i.test(String(error.stderr))) return this.info([], true, true, split)
|
||||
logger.warn(`bsdtar could not list a 7z archive: ${String(error.stderr || error.message).trim()}`)
|
||||
return null
|
||||
}
|
||||
const entries = parseBsdtarListing(stdout)
|
||||
const sample = entries.filter((e) => !e.directory).sort((a, b) => a.size - b.size)[0]
|
||||
let encrypted = false
|
||||
if (sample) {
|
||||
try {
|
||||
await run('bsdtar', ['-xOf', archive, sample.name], { timeout: 30_000, maxBuffer: 4 * 1024 * 1024, encoding: 'buffer' })
|
||||
} catch (error: any) {
|
||||
// Without the data (it's all zeros here), anything else fails too: only this message counts.
|
||||
encrypted = /content is encrypted/i.test(String(error.stderr))
|
||||
}
|
||||
}
|
||||
return this.info(entries, false, encrypted, split)
|
||||
}
|
||||
|
||||
private info(entries: { name: string; size: number; directory: boolean }[], headersEncrypted: boolean, encrypted: boolean, split: boolean): RarInfo {
|
||||
return {
|
||||
format: '7z',
|
||||
volume: split,
|
||||
firstVolume: true,
|
||||
solid: false,
|
||||
headersEncrypted,
|
||||
truncated: false,
|
||||
entries: entries.map((e) => ({
|
||||
name: e.name,
|
||||
size: e.size,
|
||||
packedSize: 0,
|
||||
directory: e.directory,
|
||||
encrypted: encrypted && !e.directory,
|
||||
splitBefore: false,
|
||||
splitAfter: false,
|
||||
stored: false,
|
||||
dataOffset: 0,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The archive's tail from the last volume: the header (checked against its
|
||||
* CRC) and, for an encoded header, the packed stream before it. `from` is
|
||||
* its position in the whole archive.
|
||||
*/
|
||||
private async readTail(last: any, start: SevenZipStart): Promise<{ from: number; data: Buffer } | null> {
|
||||
const ids = new Map(segmentsOf(last).map((s) => [s.number, s]))
|
||||
const numbers = [...ids.keys()]
|
||||
if (numbers.length === 0) return null
|
||||
const segments = new Map<number, DecodedSegment>()
|
||||
const segment = async (n: number) => {
|
||||
if (!segments.has(n)) {
|
||||
const id = ids.get(n)?.id
|
||||
if (!id) throw new Error(`part ${n} of ${last.filename} is missing`)
|
||||
segments.set(n, decodeSegment(await this.body(id), { lenientSize: true }))
|
||||
}
|
||||
return segments.get(n)!
|
||||
}
|
||||
|
||||
const lastNumber = numbers[numbers.length - 1]
|
||||
const lastSegment = await segment(lastNumber)
|
||||
const volumeSize = lastSegment.meta.begin.size
|
||||
const volumeStart = start.archiveSize - volumeSize
|
||||
const partBegin = (lastSegment.meta.part?.begin ?? 1) - 1
|
||||
const partSize = lastNumber > 1 ? partBegin / (lastNumber - 1) : volumeSize
|
||||
if (volumeStart < 0 || !Number.isInteger(partSize) || partSize <= 0) return null
|
||||
|
||||
// Bytes [from, to) of the whole archive, from the last volume's parts.
|
||||
const read = async (from: number, to: number): Promise<Buffer | null> => {
|
||||
if (from < volumeStart) return null
|
||||
const a = from - volumeStart
|
||||
const b = to - volumeStart
|
||||
const pieces: Buffer[] = []
|
||||
for (let n = Math.floor(a / partSize) + 1; n <= Math.floor((b - 1) / partSize) + 1; n++) {
|
||||
const s = await segment(n)
|
||||
const offset = (s.meta.part?.begin ?? 1) - 1
|
||||
if (offset !== (n - 1) * partSize) return null
|
||||
pieces.push(s.data.subarray(Math.max(0, a - offset), Math.max(0, Math.min(s.data.length, b - offset))))
|
||||
}
|
||||
const data = Buffer.concat(pieces)
|
||||
return data.length === b - a ? data : null
|
||||
}
|
||||
|
||||
const header = await read(start.headerStart, start.archiveSize)
|
||||
if (!header || !checkHeader(start, header)) return null
|
||||
const from = Math.min(listingStart(start, header), start.headerStart)
|
||||
if (start.archiveSize - from > MAX_TAIL_BYTES) return null
|
||||
const data = await read(from, start.archiveSize)
|
||||
return data ? { from, data } : null
|
||||
}
|
||||
|
||||
private async body(messageId: string): Promise<Buffer> {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
return await NntpService.timed(NntpService.fetchBody(conn, messageId), `BODY ${messageId}`)
|
||||
} catch (error: any) {
|
||||
failed = !/^4\d\d/.test(String(error.message))
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new SevenZipService()
|
||||
@@ -0,0 +1,80 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import { parseSfv, sfvReleaseName } from '#lib/content/index'
|
||||
|
||||
/** Releases made by name rules alone; an SFV's listing outranks them. */
|
||||
const WEAK_SOURCES = ['stem', 'weak-stem', 'subject']
|
||||
/** Files further apart than this from the SFV in posting time aren't taken. */
|
||||
const WINDOW_MS = 2 * 24 * 60 * 60 * 1000
|
||||
|
||||
/**
|
||||
* SFV grouping (CONCEPT.md D34). An uploader that posts every track of an
|
||||
* album as its own file (13.5k such releases from one poster on homelab)
|
||||
* still posts the album's SFV, which names them all.
|
||||
*/
|
||||
class SfvService {
|
||||
/**
|
||||
* Records an SFV file's listing on its release, moves the listed files that
|
||||
* sit in weak releases into it, and names the release after the album when
|
||||
* it's still named after a track or the SFV itself. Returns how many files
|
||||
* moved, or null when it isn't an SFV.
|
||||
*/
|
||||
async attach(sfvFile: any, text: string): Promise<number | null> {
|
||||
const entries = parseSfv(text)
|
||||
if (entries.length === 0 || !sfvFile.release_id) return null
|
||||
const releaseId = Number(sfvFile.release_id)
|
||||
const postedAt = sfvFile.date ? new Date(Number(sfvFile.date)) : null
|
||||
await db
|
||||
.knexQuery()
|
||||
.table('sfv_entries')
|
||||
.insert(entries.map((e) => ({ release_id: releaseId, name: e.name, crc: e.crc, poster: sfvFile.poster, posted_at: postedAt })))
|
||||
.onConflict(['release_id', 'name'])
|
||||
.ignore()
|
||||
await db.from('files').where('id', sfvFile.id).update({ content_type: 'sfv' })
|
||||
|
||||
const candidates = await db
|
||||
.from('files as f')
|
||||
.join('releases as r', 'r.id', 'f.release_id')
|
||||
.whereIn(db.raw('coalesce(f.real_name, f.filename)') as any, entries.map((e) => e.name))
|
||||
.whereNot('f.release_id', releaseId)
|
||||
.whereIn('r.source', WEAK_SOURCES)
|
||||
.select('f.id', 'f.poster', 'f.date')
|
||||
const moving = candidates.filter((f) => this.sameUpload(f, sfvFile.poster, postedAt))
|
||||
await this.move(moving.map((f) => Number(f.id)), releaseId)
|
||||
|
||||
const release = await db.from('releases').where('id', releaseId).first()
|
||||
const album = sfvReleaseName(String(sfvFile.real_name ?? sfvFile.filename))
|
||||
if (release && WEAK_SOURCES.includes(release.source) && album !== release.name && /^\d{1,3}-/.test(release.name)) {
|
||||
await db.from('releases').where('id', releaseId).update({ name: album })
|
||||
}
|
||||
logger.info(`SFV "${sfvFile.filename}": ${entries.length} files listed, ${moving.length} moved into release ${releaseId} "${album}".`)
|
||||
return moving.length
|
||||
}
|
||||
|
||||
/** A new file an SFV already listed joins that SFV's release. Returns the release, or null. */
|
||||
async claim(file: { id: number; name: string; poster?: string | null; date?: number | null; release_id?: number | null }): Promise<number | null> {
|
||||
const entries = await db.from('sfv_entries').where('name', file.name)
|
||||
const entry = entries.find((e) => this.sameUpload(file, e.poster, e.posted_at ? new Date(e.posted_at) : null))
|
||||
if (!entry || Number(entry.release_id) === Number(file.release_id)) return null
|
||||
if (file.release_id) {
|
||||
const current = await db.from('releases').where('id', file.release_id).first()
|
||||
if (current && !WEAK_SOURCES.includes(current.source)) return null
|
||||
}
|
||||
await this.move([file.id], Number(entry.release_id))
|
||||
return Number(entry.release_id)
|
||||
}
|
||||
|
||||
private sameUpload(file: { poster?: string | null; date?: number | string | null }, poster: string | null, postedAt: Date | null) {
|
||||
if (poster && file.poster && file.poster !== poster) return false
|
||||
if (postedAt && file.date && Math.abs(Number(file.date) - postedAt.getTime()) > WINDOW_MS) return false
|
||||
return true
|
||||
}
|
||||
|
||||
private async move(fileIds: number[], releaseId: number) {
|
||||
if (fileIds.length === 0) return
|
||||
await db.from('files').whereIn('id', fileIds).update({ release_id: releaseId })
|
||||
await db.from('binaries').whereIn('file_id', fileIds).update({ release_id: releaseId })
|
||||
}
|
||||
}
|
||||
|
||||
export default new SfvService()
|
||||
+270
-53
@@ -1,28 +1,55 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import NzbImportService from '#services/NzbImportService'
|
||||
import { type OverviewHeader } from '#services/IngestService'
|
||||
import IngestService, { type OverviewHeader } from '#services/IngestService'
|
||||
import { parseNzb } from '#lib/nzb/index'
|
||||
import {
|
||||
moderationTarget,
|
||||
passwordFromSpot,
|
||||
spotterId,
|
||||
decodeSpotBinary,
|
||||
decodeSpotText,
|
||||
parseSpotArticleHeaders,
|
||||
parseSpotHeader,
|
||||
parseSpotXml,
|
||||
verifySpotHeader,
|
||||
type SpotHeader,
|
||||
type SpotXml,
|
||||
} from '#lib/spotnet/index'
|
||||
|
||||
/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
|
||||
const MAX_NZB_SEGMENTS = 20
|
||||
/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
|
||||
const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
|
||||
/** Rows per INSERT: 14 columns each, under Postgres' 65,535 bind parameters. */
|
||||
const INSERT_CHUNK = 2000
|
||||
/** Spot images larger than this many articles aren't fetched. */
|
||||
const MAX_IMAGE_SEGMENTS = 10
|
||||
/** A claimed spot whose worker died is taken again after this long. */
|
||||
const RECLAIM_AFTER = '1 hour'
|
||||
|
||||
/**
|
||||
* A spot's title: from its XML, which is decoded properly (the Subject of a
|
||||
* raw header is read as ISO-8859-1), else from the header.
|
||||
*/
|
||||
function spotTitle(head: { header: SpotHeader; xml: SpotXml | null }): string {
|
||||
return head.xml?.title || head.header.title
|
||||
}
|
||||
|
||||
/**
|
||||
* A placeholder NZB (e.g. Dummy.rar + Dummy.nzb, 2.6 KB, shared by many
|
||||
* spots): less than 1% of the size the spot announces (the real NZB is
|
||||
* handed out elsewhere), or under 16 KB in all (text-only spots: requests,
|
||||
* announcements). Its files are not imported, and the spot is hidden.
|
||||
*/
|
||||
export function isDummyNzb(nzb: { files: { segments: { bytes: number }[] }[] }, spotSize: number): boolean {
|
||||
const bytes = nzb.files.reduce((sum, f) => sum + f.segments.reduce((s, x) => s + x.bytes, 0), 0)
|
||||
return bytes < 16 * 1024 || (spotSize > 0 && bytes < spotSize / 100)
|
||||
}
|
||||
|
||||
export function isSpotGroup(name: string): boolean {
|
||||
return indexerConfig.spotGroups.includes(name)
|
||||
}
|
||||
@@ -55,74 +82,115 @@ class SpotService {
|
||||
posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
|
||||
key_id: spot.keyId,
|
||||
spotter_id: spotterId(spot.selfSignedPubKey) || null,
|
||||
// From the overview; titles with accents may fail here (UTF-8 decoding)
|
||||
// and are verified again on their raw header by worker:spots.
|
||||
verified: verifySpotHeader(spot),
|
||||
created_at: now,
|
||||
})
|
||||
}
|
||||
if (rows.length === 0) return 0
|
||||
await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore()
|
||||
for (let i = 0; i < rows.length; i += INSERT_CHUNK) {
|
||||
await db.knexQuery().table('spots').insert(rows.slice(i, i + INSERT_CHUNK)).onConflict('message_id').ignore()
|
||||
}
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */
|
||||
/**
|
||||
* Takes up to `limit` pending spots, marked `processing`: recent ones
|
||||
* (to be imported) first, then the history in posting order, so live spots
|
||||
* don't wait behind a backfill and moderation follows its targets.
|
||||
*/
|
||||
async claim(limit: number): Promise<any[]> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE spots SET status = 'processing', processed_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots
|
||||
WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
|
||||
ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[limit]
|
||||
)
|
||||
const claim = async (filter: string, bindings: unknown[], n: number) =>
|
||||
(
|
||||
await db.rawQuery(
|
||||
`UPDATE spots SET status = 'processing', processed_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots WHERE ${filter}
|
||||
ORDER BY article_number LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[...bindings, n]
|
||||
)
|
||||
).rows as any[]
|
||||
const rows = await claim(`status = 'pending' AND posted_at >= ?`, [this.importCutoff()], limit)
|
||||
if (rows.length < limit) rows.push(...(await claim(`status = 'pending'`, [], limit - rows.length)))
|
||||
if (rows.length < limit) {
|
||||
rows.push(...(await claim(`status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}'`, [], limit - rows.length)))
|
||||
}
|
||||
return rows
|
||||
}
|
||||
|
||||
/** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */
|
||||
/** Spots posted before this are listed, not imported: their NZB is read on first download. */
|
||||
importCutoff(): Date {
|
||||
return new Date(Date.now() - indexerConfig.spotImportDays * 24 * 60 * 60 * 1000)
|
||||
}
|
||||
|
||||
/** Old, verified spots that aren't moderation: listed in bulk, without reading anything more. */
|
||||
isListable(spot: any): boolean {
|
||||
return (
|
||||
spot.verified === true &&
|
||||
spot.posted_at !== null &&
|
||||
new Date(spot.posted_at) < this.importCutoff() &&
|
||||
!moderationTarget({ keyId: spot.key_id, title: spot.title })
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates the listing releases (source `spot`, no files) for these spots.
|
||||
* The release key is the one an import of the spot's NZB uses, so the
|
||||
* import on first download fills this release.
|
||||
*/
|
||||
async list(spots: any[]): Promise<number> {
|
||||
if (spots.length === 0) return 0
|
||||
const keyOf = (spot: any) => createHash('sha1').update(`nzb\n${spot.message_id}`).digest('hex')
|
||||
const ids = await IngestService.upsertReleases(
|
||||
db.connection().getWriteClient(),
|
||||
spots.map((spot) => ({
|
||||
release: { key: keyOf(spot), name: spot.title, source: 'spot' as const },
|
||||
poster: spot.poster,
|
||||
groups: [spot.group_name],
|
||||
postedAt: spot.posted_at ? new Date(spot.posted_at) : undefined,
|
||||
}))
|
||||
)
|
||||
const rows = spots.map((spot) => ({ message_id: spot.message_id, release_id: ids.get(keyOf(spot)), size: spot.size }))
|
||||
await db.rawQuery(
|
||||
`UPDATE releases r SET size = v.size FROM jsonb_to_recordset(?::jsonb) AS v(release_id bigint, size bigint)
|
||||
WHERE r.id = v.release_id AND r.file_count = 0 AND v.size IS NOT NULL`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE spots s SET status = 'listed', release_id = v.release_id, processed_at = now()
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(message_id text, release_id bigint)
|
||||
WHERE s.message_id = v.message_id`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/**
|
||||
* Verifies the spot on its raw header (the signature covers ISO-8859-1
|
||||
* bytes, which the overview, read as UTF-8, may have mangled), then
|
||||
* applies moderation, lists an old spot or imports a recent one's NZB.
|
||||
* Returns the new status.
|
||||
*/
|
||||
async process(spot: any): Promise<string> {
|
||||
const update: Record<string, unknown> = { processed_at: new Date(), error: null }
|
||||
try {
|
||||
const raw = await this.fetch(`HEAD <${spot.message_id}>`)
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
|
||||
// The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled.
|
||||
const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date)
|
||||
if (!header || !verifySpotHeader(header)) {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) {
|
||||
update.status = 'unverified'
|
||||
} else if (moderationTarget(header)) {
|
||||
} else if (moderationTarget(head.header)) {
|
||||
update.status = 'moderation'
|
||||
await this.moderate(header, moderationTarget(header)!)
|
||||
await this.moderate(head.header, moderationTarget(head.header)!)
|
||||
} else {
|
||||
const parsed = parseSpotXml(xml)
|
||||
update.title = header.title
|
||||
update.poster = header.poster
|
||||
update.spotter_id = spotterId(header.selfSignedPubKey) || null
|
||||
if (parsed?.website) update.website = parsed.website
|
||||
const segments = parsed?.nzbSegments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) {
|
||||
update.status = 'no-nzb'
|
||||
update.title = spotTitle(head)
|
||||
update.poster = head.header.poster
|
||||
update.spotter_id = spotterId(head.header.selfSignedPubKey) || null
|
||||
Object.assign(update, this.details(head.xml))
|
||||
if (new Date(head.header.stamp * 1000) < this.importCutoff()) {
|
||||
await this.list([{ ...spot, title: spotTitle(head), poster: head.header.poster }])
|
||||
update.status = 'listed'
|
||||
} else {
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
const nzbXml = decodeSpotBinary(bodies).toString('utf8')
|
||||
const nzb = parseNzb(nzbXml)
|
||||
if (!nzb || nzb.files.length === 0) {
|
||||
update.status = 'no-nzb'
|
||||
} else {
|
||||
const groups = [...new Set(nzb.files.flatMap((f) => f.groups))]
|
||||
const result = await NzbImportService.import(
|
||||
{
|
||||
id: null,
|
||||
filename: `${header.title}.nzb`,
|
||||
poster: header.poster,
|
||||
groups,
|
||||
date: header.stamp * 1000,
|
||||
message_ids: { '1': { id: spot.message_id } },
|
||||
},
|
||||
nzbXml,
|
||||
{ name: header.title, source: 'spot', moveContainer: false }
|
||||
)
|
||||
update.status = result ? 'imported' : 'no-nzb'
|
||||
if (result) update.release_id = result.releaseId
|
||||
}
|
||||
Object.assign(update, await this.importNzb(spot.message_id, head))
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
@@ -131,9 +199,149 @@ class SpotService {
|
||||
logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
if (update.status === 'imported' || update.status === 'listed') await this.applyPassword(spot.message_id)
|
||||
return update.status as string
|
||||
}
|
||||
|
||||
/**
|
||||
* The NZB of a listing, on first download: reads and imports it. Returns
|
||||
* the release that now holds its files (the listing's own, unless the NZB
|
||||
* joined files already known elsewhere), or null.
|
||||
*/
|
||||
async importListing(releaseId: number): Promise<number | null> {
|
||||
const spot = await db.from('spots').where('release_id', releaseId).where('status', 'listed').first()
|
||||
if (!spot) return null
|
||||
return this.reimport(spot)
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads a spot's NZB again and imports it, so its files end up in the
|
||||
* spot's own release under the spot's title. Returns that release, or null.
|
||||
*/
|
||||
async reimport(spot: any): Promise<number | null> {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
if (!head) return null
|
||||
const result = await this.importNzb(spot.message_id, head)
|
||||
await db
|
||||
.from('spots')
|
||||
.where('message_id', spot.message_id)
|
||||
.update({ ...result, ...this.details(head.xml), processed_at: new Date() })
|
||||
await this.applyPassword(spot.message_id)
|
||||
return (result.release_id as number | undefined) ?? null
|
||||
}
|
||||
|
||||
/**
|
||||
* A password the spotter wrote in the spot's title or description (not in
|
||||
* the NZB) goes on its release, unless the release has one already.
|
||||
*/
|
||||
async applyPassword(messageId: string): Promise<string | null> {
|
||||
const spot = await db.from('spots').where('message_id', messageId).whereNotNull('release_id').first()
|
||||
const password = spot ? passwordFromSpot(spot.title, spot.description) : null
|
||||
if (!password) return null
|
||||
await db
|
||||
.from('releases')
|
||||
.where('id', spot.release_id)
|
||||
.whereNull('password')
|
||||
.update({ password, password_source: 'spot' })
|
||||
return password
|
||||
}
|
||||
|
||||
/** Columns for what the spot's XML adds to its overview: description, image, website. */
|
||||
private details(xml: SpotXml | null): Record<string, unknown> {
|
||||
return {
|
||||
description: xml?.description || null,
|
||||
image: xml?.image ? JSON.stringify(xml.image) : null,
|
||||
...(xml?.website ? { website: xml.website } : {}),
|
||||
details_at: new Date(),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A spot row with its description and image, reading its full header
|
||||
* first when that hasn't happened yet (old spots are listed from their
|
||||
* overview only). Returns the row as it is when Usenet can't be reached.
|
||||
*/
|
||||
async withDetails(spot: any): Promise<any> {
|
||||
if (spot.details_at) return spot
|
||||
try {
|
||||
const head = await this.readHead(spot.message_id)
|
||||
const update = head ? this.details(head.xml) : { details_at: new Date() }
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
await this.applyPassword(spot.message_id)
|
||||
return { ...spot, ...update, image: head?.xml?.image ?? null }
|
||||
} catch (error: any) {
|
||||
logger.warn(`Spot ${spot.message_id} details: ${error.message}`)
|
||||
// The article is gone (430): nothing to read later either.
|
||||
if (/^4\d\d/.test(String(error.message))) {
|
||||
await db.from('spots').where('message_id', spot.message_id).update({ details_at: new Date() })
|
||||
}
|
||||
return spot
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the details of up to `limit` imported spots that don't have them
|
||||
* (imported before details were kept), newest first. Returns how many.
|
||||
* Listed spots are left to their first view: there are millions.
|
||||
* Returns how many were read (0 when Usenet can't be reached).
|
||||
*/
|
||||
async fillDetails(limit: number): Promise<number> {
|
||||
const spots = await db
|
||||
.from('spots')
|
||||
.where('status', 'imported')
|
||||
.whereNull('details_at')
|
||||
.orderBy('posted_at', 'desc')
|
||||
.limit(limit)
|
||||
let read = 0
|
||||
for (const spot of spots) if ((await this.withDetails(spot)).details_at) read++
|
||||
return read
|
||||
}
|
||||
|
||||
/** The spot's image as posted on Usenet (Spotnet posts them uncompressed), or null. */
|
||||
async image(spot: any): Promise<Buffer | null> {
|
||||
const segments: string[] = spot.image?.segments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_IMAGE_SEGMENTS) return null
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
return decodeSpotBinary(bodies, false)
|
||||
}
|
||||
|
||||
/** The spot's raw header, verified; null when it doesn't verify. */
|
||||
private async readHead(messageId: string): Promise<{ header: SpotHeader; xml: SpotXml | null } | null> {
|
||||
const raw = await this.fetch(`HEAD <${messageId}>`)
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
|
||||
const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', messageId, headers.date)
|
||||
if (!header || !verifySpotHeader(header)) return null
|
||||
return { header, xml: parseSpotXml(decodeSpotText(xml)) }
|
||||
}
|
||||
|
||||
/** Downloads the spot's compressed NZB and imports it as a release (D26). */
|
||||
private async importNzb(messageId: string, head: { header: SpotHeader; xml: SpotXml | null }): Promise<Record<string, unknown>> {
|
||||
const segments = head.xml?.nzbSegments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) return { status: 'no-nzb' }
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
const nzbXml = decodeSpotBinary(bodies).toString('utf8')
|
||||
const nzb = parseNzb(nzbXml)
|
||||
if (!nzb || nzb.files.length === 0) return { status: 'no-nzb' }
|
||||
const { header } = head
|
||||
if (isDummyNzb(nzb, header.size)) return { status: 'dummy' }
|
||||
const title = spotTitle(head)
|
||||
const result = await NzbImportService.import(
|
||||
{
|
||||
id: null,
|
||||
filename: `${title}.nzb`,
|
||||
poster: header.poster,
|
||||
groups: [...new Set(nzb.files.flatMap((f) => f.groups))],
|
||||
date: header.stamp * 1000,
|
||||
first_message_id: messageId,
|
||||
},
|
||||
nzbXml,
|
||||
{ name: title, source: 'spot', moveContainer: false, trusted: true }
|
||||
)
|
||||
return result ? { status: 'imported', release_id: result.releaseId } : { status: 'no-nzb' }
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a delete/dispose/remove: the target spot is marked `removed`
|
||||
* (its release is then hidden from search). As in Spotweb, a personal
|
||||
@@ -150,6 +358,15 @@ class SpotService {
|
||||
}
|
||||
await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
|
||||
logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
|
||||
// Spots that joined its release (a correction, a repost) take the files over now.
|
||||
if (target.release_id) {
|
||||
const sharing = await db
|
||||
.from('spots')
|
||||
.where('release_id', target.release_id)
|
||||
.where('status', 'imported')
|
||||
.whereNot('message_id', targetId)
|
||||
for (const spot of sharing) await this.reimport(spot)
|
||||
}
|
||||
}
|
||||
|
||||
/** One HEAD or BODY, as raw bytes. */
|
||||
@@ -157,7 +374,7 @@ class SpotService {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
const { data } = await conn._artcmd(command)
|
||||
const { data } = await NntpService.timed<any>(conn._artcmd(command), command)
|
||||
return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
|
||||
} catch (error: any) {
|
||||
// 430 (no such article) is an answer, not a broken connection.
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import IngestService, { type IngestSegment } from '#services/IngestService'
|
||||
import { counterFileKey } from '#lib/subject/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
|
||||
/** Rows per statement (a key and a JSON array each). */
|
||||
const CHUNK = 5000
|
||||
|
||||
/**
|
||||
* Staging of lone header segments (CONCEPT.md D35). Uploads that give every
|
||||
* article its own random subject turned each article into a binary, a part
|
||||
* and a release, which GC deleted two hours later: ~3.9M binaries an hour
|
||||
* against ~123k real files. A segment whose file is new and alone in its
|
||||
* fetch batch now waits in `staged_segments` (unlogged, one index) until a
|
||||
* second segment of that file arrives; only then do both go into the real
|
||||
* tables. GC drops what waited longer than STAGING_TTL_SECONDS (30 min).
|
||||
*
|
||||
* Counter grouping (D36): those random-subject uploads still keep poster,
|
||||
* file counter `[i/M]` and part count per file. Staged segments also carry
|
||||
* that key (`counter_key`); once two staged segments from different subjects
|
||||
* share it, they become one file under that key, and later segments with it
|
||||
* join that file directly.
|
||||
*/
|
||||
class StagingService {
|
||||
enabled(groupName: string): boolean {
|
||||
const groups = indexerConfig.stagingGroups
|
||||
return groups.includes('*') || groups.includes(groupName)
|
||||
}
|
||||
|
||||
/** Whether counter grouping (D36) applies to this group. */
|
||||
countersEnabled(groupName: string): boolean {
|
||||
const groups = indexerConfig.counterGroupingGroups
|
||||
return groups.includes('*') || groups.includes(groupName)
|
||||
}
|
||||
|
||||
/**
|
||||
* The segments of a header batch that should be ingested now: the batch's
|
||||
* own, minus lone segments of new files (staged instead), plus staged
|
||||
* segments whose file this batch shows to be real. `staged` counts the
|
||||
* segments that went to staging.
|
||||
*/
|
||||
async filter(segments: IngestSegment[], { counters = false } = {}): Promise<{ ingest: IngestSegment[]; staged: number }> {
|
||||
const byKey = new Map<string, IngestSegment[]>()
|
||||
for (const segment of segments) {
|
||||
const list = byKey.get(segment.fileKey)
|
||||
if (list) list.push(segment)
|
||||
else byKey.set(segment.fileKey, [segment])
|
||||
}
|
||||
|
||||
const ingest: IngestSegment[] = []
|
||||
const lone: IngestSegment[] = []
|
||||
for (const list of byKey.values()) {
|
||||
// Single-segment files are complete; two different articles of a file in one batch make it real.
|
||||
// (One article twice is a crosspost, LOG.md 2026-10-02.)
|
||||
const articles = new Set(list.map((s) => s.messageId)).size
|
||||
if (list[0].totalParts <= 1 || articles >= 2 || list[0].weakIdentity || list[0].size != null) ingest.push(...list)
|
||||
else lone.push(list[0])
|
||||
}
|
||||
|
||||
const counterOf = new Map<IngestSegment, string>()
|
||||
if (counters) {
|
||||
for (const segment of lone) {
|
||||
const key = counterFileKey(segment)
|
||||
if (key) counterOf.set(segment, key)
|
||||
}
|
||||
}
|
||||
|
||||
// Files we already have a binary for (by subject, or by counters) take their segments directly.
|
||||
const known = new Set<string>()
|
||||
const lookup = [...new Set([...lone.map((s) => s.fileKey), ...counterOf.values()])]
|
||||
for (let i = 0; i < lookup.length; i += CHUNK) {
|
||||
const rows = await db.from('binaries').whereIn('file_key', lookup.slice(i, i + CHUNK)).select('file_key')
|
||||
for (const row of rows) known.add(row.file_key)
|
||||
}
|
||||
const toStage: IngestSegment[] = []
|
||||
const knownCounters = new Set<string>()
|
||||
for (const segment of lone) {
|
||||
const counter = counterOf.get(segment)
|
||||
if (known.has(segment.fileKey)) {
|
||||
ingest.push(segment)
|
||||
} else if (counter && known.has(counter)) {
|
||||
ingest.push(asCounterFile(segment, counter))
|
||||
knownCounters.add(counter)
|
||||
} else {
|
||||
toStage.push(segment)
|
||||
}
|
||||
}
|
||||
|
||||
// Stage the rest; a file that now has two staged segments comes out again.
|
||||
const promoted: string[] = []
|
||||
// Key order, so concurrent jobs lock rows in the same order.
|
||||
toStage.sort((a, b) => a.fileKey.localeCompare(b.fileKey))
|
||||
for (let i = 0; i < toStage.length; i += CHUNK) {
|
||||
const rows = toStage.slice(i, i + CHUNK).map((s) => ({
|
||||
file_key: s.fileKey,
|
||||
counter_key: counterOf.get(s) ?? null,
|
||||
segments: [serialize(s)],
|
||||
staged_at: new Date(),
|
||||
}))
|
||||
const { rows: result } = await retryDeadlock(() => db.rawQuery(
|
||||
`INSERT INTO staged_segments (file_key, counter_key, segments, staged_at)
|
||||
SELECT file_key, counter_key, segments, staged_at
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(file_key text, counter_key text, segments jsonb, staged_at timestamptz)
|
||||
ON CONFLICT (file_key) DO UPDATE SET segments = CASE
|
||||
-- The same article again (a crosspost, or a re-fetched range) is no second segment.
|
||||
WHEN EXISTS (SELECT 1 FROM jsonb_array_elements(staged_segments.segments) e
|
||||
WHERE e->>'messageId' = EXCLUDED.segments->0->>'messageId') THEN staged_segments.segments
|
||||
ELSE staged_segments.segments || EXCLUDED.segments END
|
||||
RETURNING file_key, jsonb_array_length(segments) AS n`,
|
||||
[JSON.stringify(rows)]
|
||||
))
|
||||
for (const row of result) if (Number(row.n) >= 2) promoted.push(row.file_key)
|
||||
}
|
||||
|
||||
// Counter files: two staged rows (two subjects) with one counter key make a file.
|
||||
const stagedCounters = [...new Set(toStage.map((s) => counterOf.get(s)).filter((k): k is string => Boolean(k)))]
|
||||
const readyCounters: string[] = []
|
||||
for (let i = 0; i < stagedCounters.length; i += CHUNK) {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT counter_key FROM staged_segments WHERE counter_key = ANY(?) GROUP BY counter_key HAVING count(*) >= 2`,
|
||||
[stagedCounters.slice(i, i + CHUNK)]
|
||||
)
|
||||
for (const row of rows) readyCounters.push(row.counter_key)
|
||||
}
|
||||
|
||||
// Promoted files, staged segments of files this batch ingests anyway, and ready counter files.
|
||||
const fileKeys = [...new Set([...promoted, ...ingest.map((s) => s.fileKey)])]
|
||||
const counterKeys = [...new Set([...readyCounters, ...knownCounters])]
|
||||
const seen = new Set(ingest.map((s) => s.messageId))
|
||||
for (const segment of await this.take(fileKeys, counterKeys)) {
|
||||
if (!seen.has(segment.messageId)) ingest.push(segment)
|
||||
seen.add(segment.messageId)
|
||||
}
|
||||
const staged = toStage.filter((s) => !seen.has(s.messageId)).length
|
||||
return { ingest, staged }
|
||||
}
|
||||
|
||||
/**
|
||||
* GC: drops staged segments that waited longer than `maxAgeSeconds`, then
|
||||
* ingests those whose file got a binary meanwhile (two fetch jobs racing).
|
||||
* Expiry comes first so the table stays bounded whatever the budget; the
|
||||
* adoption starts from recently seen binaries (index on last_seen_at,
|
||||
* which moves in 10-minute steps) rather than joining every staged row.
|
||||
*/
|
||||
async sweep(maxAgeSeconds: number, budget: number): Promise<{ adopted: number; expired: number }> {
|
||||
let adopted = 0
|
||||
let expired = 0
|
||||
while (Date.now() < budget) {
|
||||
const result = await db.rawQuery(
|
||||
`DELETE FROM staged_segments WHERE ctid IN (
|
||||
SELECT ctid FROM staged_segments WHERE staged_at < now() - make_interval(secs => ?) LIMIT 20000)`,
|
||||
[maxAgeSeconds]
|
||||
)
|
||||
expired += result.rowCount
|
||||
if (result.rowCount < 20000) break
|
||||
}
|
||||
let lastId = 0
|
||||
while (Date.now() < budget) {
|
||||
const recent = await db
|
||||
.from('binaries')
|
||||
.where('last_seen_at', '>', new Date(Date.now() - (maxAgeSeconds + 15 * 60) * 1000))
|
||||
.where('id', '>', lastId)
|
||||
.orderBy('id')
|
||||
.limit(CHUNK)
|
||||
.select('id', 'file_key')
|
||||
if (recent.length === 0) break
|
||||
lastId = Number(recent[recent.length - 1].id)
|
||||
const keys = recent.map((b) => b.file_key as string)
|
||||
// A binary's key is a subject key or a counter key; staged rows are looked up by both.
|
||||
const segments = await this.take(keys, keys)
|
||||
if (segments.length > 0) await IngestService.ingest(segments)
|
||||
adopted += segments.length
|
||||
}
|
||||
return { adopted, expired }
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes and returns the staged segments of these files: by subject key,
|
||||
* or by counter key (returned as segments of the counter file).
|
||||
*/
|
||||
private async take(fileKeys: string[], counterKeys: string[] = []): Promise<IngestSegment[]> {
|
||||
const taken: IngestSegment[] = []
|
||||
for (let i = 0; i < fileKeys.length; i += CHUNK) {
|
||||
const { rows } = await retryDeadlock(() =>
|
||||
db.rawQuery(`DELETE FROM staged_segments WHERE file_key = ANY(?) RETURNING segments`, [fileKeys.slice(i, i + CHUNK)])
|
||||
)
|
||||
for (const row of rows) taken.push(...(row.segments as any[]).map(deserialize))
|
||||
}
|
||||
for (let i = 0; i < counterKeys.length; i += CHUNK) {
|
||||
const { rows } = await retryDeadlock(() =>
|
||||
db.rawQuery(`DELETE FROM staged_segments WHERE counter_key = ANY(?) RETURNING counter_key, segments`, [counterKeys.slice(i, i + CHUNK)])
|
||||
)
|
||||
for (const row of rows) taken.push(...(row.segments as any[]).map((s) => asCounterFile(deserialize(s), row.counter_key)))
|
||||
}
|
||||
return taken
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A segment as part of its counter file (D36): the counter key is its file
|
||||
* key, it is a weak identity (a repeated part number from another upload
|
||||
* marks the binary conflicted), and its release is a weak one of its own,
|
||||
* which the upload's par2 set can gather later.
|
||||
*/
|
||||
function asCounterFile(segment: IngestSegment, counterKey: string): IngestSegment {
|
||||
return {
|
||||
...segment,
|
||||
fileKey: counterKey,
|
||||
weakIdentity: true,
|
||||
release: {
|
||||
key: createHash('sha1').update(`counter-release\n${counterKey}`).digest('hex'),
|
||||
name: fileStem(segment.name),
|
||||
source: 'weak-stem',
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a staging statement again when it lost a deadlock: concurrent fetch
|
||||
* jobs insert by file key and delete by counter key, so their row locks can
|
||||
* cross. Postgres rolled the statement back, and each one is safe to repeat.
|
||||
*/
|
||||
async function retryDeadlock<T>(statement: () => Promise<T>, attempts = 5): Promise<T> {
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
return await statement()
|
||||
} catch (error: any) {
|
||||
if (error?.code !== '40P01' || attempt >= attempts) throw error
|
||||
await new Promise((resolve) => setTimeout(resolve, 50 * attempt + Math.random() * 100))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function serialize(segment: IngestSegment) {
|
||||
return {
|
||||
...segment,
|
||||
postedAt: segment.postedAt?.toISOString(),
|
||||
articleNumber: segment.articleNumber?.toString(),
|
||||
}
|
||||
}
|
||||
|
||||
function deserialize(stored: any): IngestSegment {
|
||||
return {
|
||||
...stored,
|
||||
postedAt: stored.postedAt ? new Date(stored.postedAt) : undefined,
|
||||
articleNumber: stored.articleNumber ?? undefined,
|
||||
}
|
||||
}
|
||||
|
||||
export default new StagingService()
|
||||
Reference in new issue
Block a user