implemented a lot of features
This commit is contained in:
Daan Meijer committed
1 parent
62f1767df3
commit
bcb9be5da9
141 files changed
+10705
-385
No files matched your search
@@ -0,0 +1,160 @@
|
||||
import { execFile } from 'node:child_process'
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { promisify } from 'node:util'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { storedEntryData, type RarEntry, type RarInfo } from '#lib/rar/index'
|
||||
import { fileStem, rarReleaseIdentity } from '#lib/release/index'
|
||||
|
||||
/** Stored per file and release; a listing of thousands of files is not useful. */
|
||||
const MAX_ENTRIES = 500
|
||||
|
||||
export interface ArchiveResult {
|
||||
info: RarInfo
|
||||
releaseId?: number
|
||||
releaseRenamed?: string
|
||||
grouped?: boolean
|
||||
}
|
||||
|
||||
/**
|
||||
* Records what a RAR volume's headers say: the listing on the file, the
|
||||
* release it belongs to (from the shared directory or continued entry, when
|
||||
* the file has none yet), and for first volumes the contents and password
|
||||
* flag on the release. See CONCEPT.md (D19, D21).
|
||||
*/
|
||||
class ArchiveService {
|
||||
async record(file: any, binary: any, info: RarInfo): Promise<ArchiveResult> {
|
||||
const stored = { ...info, entries: info.entries.slice(0, MAX_ENTRIES).map(({ stored: _s, dataOffset: _d, ...entry }) => entry) }
|
||||
await db.from('files').where('id', file.id).update({ archive: JSON.stringify(stored) })
|
||||
|
||||
let releaseId = file.release_id ?? binary?.release_id ?? null
|
||||
let grouped = false
|
||||
const identity = rarReleaseIdentity(info.entries)
|
||||
if (!releaseId && identity) {
|
||||
const ids = await IngestService.upsertReleases(db.connection().getWriteClient(), [
|
||||
{
|
||||
release: identity,
|
||||
poster: binary?.poster ?? file.poster,
|
||||
weakIdentity: true,
|
||||
groups: binary?.groups ?? file.groups ?? [],
|
||||
postedAt: binary?.posted_at ? new Date(binary.posted_at) : undefined,
|
||||
},
|
||||
])
|
||||
releaseId = ids.get(identity.key)!
|
||||
grouped = true
|
||||
if (binary) await db.from('binaries').where('id', binary.id).update({ release_id: releaseId })
|
||||
await db.from('files').where('id', file.id).update({ release_id: releaseId })
|
||||
}
|
||||
if (!releaseId) return { info }
|
||||
|
||||
const release = await db.from('releases').where('id', releaseId).first()
|
||||
const update: Record<string, unknown> = {}
|
||||
if (info.firstVolume !== false) {
|
||||
update.encrypted = info.headersEncrypted || info.entries.some((e) => e.encrypted)
|
||||
if (info.entries.length > 0) update.contents = JSON.stringify(stored.entries)
|
||||
} else if (info.headersEncrypted) {
|
||||
update.encrypted = true
|
||||
}
|
||||
const better = releaseNameFromContents(info)
|
||||
const renamed = release && better && (looksRandom(release.name) || release.source === 'post') ? better : undefined
|
||||
if (renamed) update.name = renamed
|
||||
if (Object.keys(update).length > 0) await db.from('releases').where('id', releaseId).update(update)
|
||||
|
||||
if (renamed) logger.info(`Release ${releaseId} renamed "${release.name}" → "${renamed}" from its archive.`)
|
||||
return { info, releaseId, releaseRenamed: renamed, grouped }
|
||||
}
|
||||
}
|
||||
|
||||
const run = promisify(execFile)
|
||||
|
||||
/** Tools that print one archive entry to stdout, tried in order. */
|
||||
const EXTRACTORS: [string, (archive: string, entry: string) => string[]][] = [
|
||||
['bsdtar', (archive, entry) => ['-xOf', archive, entry]],
|
||||
['unrar', (archive, entry) => ['p', '-inul', '-p-', '--', archive, entry]],
|
||||
]
|
||||
|
||||
/**
|
||||
* The unpacked data of one entry of a complete (single-volume) archive:
|
||||
* copied out when stored, else unpacked with bsdtar (libarchive, in the
|
||||
* Docker image) or unrar. Null when that is not possible.
|
||||
*/
|
||||
export async function extractEntry(archive: Buffer, entry: RarEntry, maxBytes = 64 * 1024 * 1024): Promise<Buffer | null> {
|
||||
const stored = storedEntryData(archive, entry)
|
||||
if (stored) return stored
|
||||
if (entry.encrypted || entry.directory || entry.splitBefore || entry.splitAfter) return null
|
||||
if (entry.size !== null && entry.size > maxBytes) return null
|
||||
|
||||
const dir = await mkdtemp(path.join(tmpdir(), 'rar-'))
|
||||
try {
|
||||
const file = path.join(dir, 'archive.rar')
|
||||
await writeFile(file, archive)
|
||||
for (const [tool, args] of EXTRACTORS) {
|
||||
try {
|
||||
const { stdout } = await run(tool, args(file, entry.name), { encoding: 'buffer', maxBuffer: maxBytes, timeout: 60_000 })
|
||||
if (stdout.length > 0) return stdout
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') logger.warn(`${tool} could not extract "${entry.name}": ${error.message}`)
|
||||
}
|
||||
}
|
||||
return null
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A release name from an archive listing: the top-level directory shared by
|
||||
* all entries when it reads like a release name, otherwise the stem of the
|
||||
* largest entry with a readable name (e.g. an archive holding just
|
||||
* `Joker.Folie.a.Deux.2024.2160p….nzb`). Samples are skipped.
|
||||
*/
|
||||
export function releaseNameFromContents(info: Pick<RarInfo, 'entries'>): string | undefined {
|
||||
const entries = info.entries.filter((e) => !e.directory)
|
||||
const tops = new Set(info.entries.map((e) => e.name.split(/[\\/]/)[0]))
|
||||
if (tops.size === 1 && info.entries.some((e) => /[\\/]/.test(e.name))) {
|
||||
const [top] = tops
|
||||
if (/[A-Za-z]{2,}.*[._-].*[A-Za-z0-9]{2,}/.test(top) && !looksRandom(top)) return top
|
||||
}
|
||||
const candidates = entries
|
||||
.map((e) => ({ stem: fileStem(e.name.split(/[\\/]/).pop() ?? e.name), size: e.size ?? 0 }))
|
||||
.filter(({ stem }) => !/(^|[._ -])sample([._ -]|$)/i.test(stem))
|
||||
.filter(({ stem }) => /[A-Za-z]{2,}.*[._ -].*[A-Za-z0-9]{2,}/.test(stem) && !looksRandom(stem))
|
||||
.sort((a, b) => b.size - a.size)
|
||||
return candidates[0]?.stem
|
||||
}
|
||||
|
||||
/**
|
||||
* Renames random-named releases from their stored archive listing (no
|
||||
* downloads). Used by `release:rename`.
|
||||
*/
|
||||
export async function renameFromStoredContents(): Promise<number> {
|
||||
const releases = await db
|
||||
.from('releases')
|
||||
.whereNotNull('contents')
|
||||
.whereRaw("jsonb_array_length(contents) > 0")
|
||||
.select('id', 'name', 'source', 'contents')
|
||||
let renamed = 0
|
||||
for (const release of releases) {
|
||||
if (!looksRandom(release.name) && release.source !== 'post') continue
|
||||
const better = releaseNameFromContents({ entries: release.contents })
|
||||
if (!better || better === release.name) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
renamed++
|
||||
}
|
||||
return renamed
|
||||
}
|
||||
|
||||
/** Obfuscated names: one long token of mixed letters and digits, no separators. */
|
||||
export function looksRandom(name: string): boolean {
|
||||
// An unknown short extension (e.g. ".bin") says nothing about the name.
|
||||
name = name.replace(/\.[A-Za-z0-9]{1,4}$/, '')
|
||||
if (/^[0-9a-f]{16,}$/i.test(name)) return true
|
||||
if (!/^[A-Za-z0-9]{16,}$/.test(name) || !/[a-z]/.test(name) || !/[A-Z]/.test(name)) return false
|
||||
// Digits, or many capitals without separators (e.g. RmISakytFwXSCEzjVEpaKk).
|
||||
return /\d/.test(name) || (name.match(/[A-Z]/g) ?? []).length >= 6
|
||||
}
|
||||
|
||||
export default new ArchiveService()
|
||||
@@ -0,0 +1,187 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import NntpService from '#services/NntpService'
|
||||
import Par2Service, { type Par2Result } from '#services/Par2Service'
|
||||
import ArchiveService, { extractEntry, looksRandom, type ArchiveResult } from '#services/ArchiveService'
|
||||
import NameService from '#services/NameService'
|
||||
import { decodeSegment, YencCrcError, YencError, YencFile } from '#lib/yenc/index'
|
||||
import NzbImportService, { type NzbImportResult } from '#services/NzbImportService'
|
||||
import { sniffContentType, type ContentType } from '#lib/content/index'
|
||||
import { parseRarHeaders, type RarEntry, type RarInfo } from '#lib/rar/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
import { looksLikeFilename } from '#lib/subject/index'
|
||||
|
||||
const FIRST_CHUNK = 16 * 1024
|
||||
|
||||
/**
|
||||
* MD5 of the first 16 KiB of a file, as par2 records it, from the body of
|
||||
* the file's first part. Undefined when that part is shorter than 16 KiB but
|
||||
* the file is not (the chunk would span two parts).
|
||||
*/
|
||||
export function firstChunkMd5(body: Buffer, fileSize: number): string | undefined {
|
||||
return chunkMd5(decodeSegment(body).data, fileSize)
|
||||
}
|
||||
|
||||
function chunkMd5(data: Buffer, fileSize: number): string | undefined {
|
||||
if (data.length < Math.min(FIRST_CHUNK, fileSize)) return undefined
|
||||
return createHash('md5').update(data.subarray(0, FIRST_CHUNK)).digest('hex')
|
||||
}
|
||||
|
||||
export interface ContentResult {
|
||||
par2?: Par2Result
|
||||
archive?: ArchiveResult
|
||||
named: number
|
||||
contentType: ContentType | null
|
||||
nzb?: NzbImportResult
|
||||
/** NZBs read from inside an archive. */
|
||||
archiveNzbs?: NzbImportResult[]
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes names and groupings from file contents (CONCEPT.md D21). For a
|
||||
* completed file with a random name, one article (the first) is enough:
|
||||
* - its fingerprint (size + first-16-KiB MD5) matches it to a par2 entry,
|
||||
* - par2 files are recognised by content and name every file of their set,
|
||||
* - RAR headers give the release grouping, name and contents.
|
||||
*/
|
||||
class ContentService {
|
||||
/** Random or missing names, and named first RAR volumes (for contents and password flag). */
|
||||
isCandidate(name: string) {
|
||||
if (!looksLikeFilename(name) || looksRandom(fileStem(name))) return true
|
||||
if (/\.nzb$/i.test(name)) return true
|
||||
return /\.part0*1\.rar$/i.test(name) || (/\.rar$/i.test(name) && !/\.part\d+\.rar$/i.test(name))
|
||||
}
|
||||
|
||||
/** Downloads a whole (small) file whose first segment is already decoded. */
|
||||
private async readWhole(first: Buffer, messageIds: Record<string, { id: string; size: number }>): Promise<Buffer | null> {
|
||||
const parts = Object.entries(messageIds).sort(([a], [b]) => Number(a) - Number(b))
|
||||
const bytes = parts.reduce((sum, [, p]) => sum + (p.size ?? 0), 0)
|
||||
if (parts.length > WHOLE_MAX_PARTS || bytes > WHOLE_MAX_BYTES) return null
|
||||
if (parts.length === 1) return first
|
||||
|
||||
let assembled: YencFile | undefined
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
for (const [, part] of parts) {
|
||||
const segment = decodeSegment(await NntpService.fetchBody(conn, part.id), { lenientSize: true })
|
||||
assembled ??= YencFile.fromSegment(segment)
|
||||
assembled.add(segment)
|
||||
}
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
return assembled!.getBuffer()
|
||||
}
|
||||
|
||||
/**
|
||||
* Downloads a small archive that lists NZBs and imports each of them
|
||||
* (CONCEPT.md D28): NZBs are posted inside RARs too, e.g. for obfuscated
|
||||
* uploads by other posters.
|
||||
*/
|
||||
private async importArchiveNzbs(file: any, first: Buffer): Promise<NzbImportResult[]> {
|
||||
const archive = await this.readWhole(first, file.message_ids)
|
||||
const info = archive && parseRarHeaders(archive)
|
||||
if (!archive || !info) return []
|
||||
const entries = info.entries.filter(isNzbEntry)
|
||||
const results: NzbImportResult[] = []
|
||||
for (const entry of entries) {
|
||||
const xml = await extractEntry(archive, entry)
|
||||
if (!xml) continue
|
||||
const imported = await NzbImportService.import(file, xml.toString('utf8'), {
|
||||
entryName: entry.name,
|
||||
moveContainer: entries.length === 1,
|
||||
})
|
||||
if (imported) results.push(imported)
|
||||
}
|
||||
return results
|
||||
}
|
||||
|
||||
async inspect(fileId: number): Promise<ContentResult | null> {
|
||||
const file = await db.from('files').where('id', fileId).first()
|
||||
const binary = await db.from('binaries').where('file_id', fileId).first()
|
||||
if (!file || !this.isCandidate(file.filename)) return null
|
||||
const first = file.message_ids?.['1']
|
||||
if (!first?.id) return null
|
||||
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let body: Buffer
|
||||
try {
|
||||
body = await NntpService.fetchBody(conn, first.id)
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
// Some tools declare a size that leaves out a header (nzbe); the CRC still has to match.
|
||||
let segment
|
||||
try {
|
||||
segment = decodeSegment(body)
|
||||
} catch (error) {
|
||||
if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
|
||||
segment = decodeSegment(body, { lenientSize: true })
|
||||
}
|
||||
const size = segment.meta.begin.size
|
||||
const contentType = sniffContentType(segment.data)
|
||||
await db.from('files').where('id', fileId).update({ content_type: contentType })
|
||||
|
||||
// Fingerprint, so par2 sets (now or later) can name this file exactly.
|
||||
if (binary) {
|
||||
await db
|
||||
.from('binaries')
|
||||
.where('id', binary.id)
|
||||
.update({
|
||||
size: db.raw('coalesce(size, ?)', [size]),
|
||||
md5_16k: db.raw('coalesce(md5_16k, ?)', [chunkMd5(segment.data, size) ?? null]),
|
||||
})
|
||||
}
|
||||
|
||||
const result: ContentResult = { named: 0, contentType }
|
||||
if (contentType === 'nzbe' || contentType === 'rclone') {
|
||||
// An encrypted NZB, or rclone crypt (names and data encrypted): nothing
|
||||
// to read, and useless without the key.
|
||||
const releaseId = file.release_id ?? binary?.release_id
|
||||
if (releaseId) {
|
||||
await db.from('releases').where('id', releaseId).update({ container: contentType, encrypted: true })
|
||||
}
|
||||
return result
|
||||
}
|
||||
if (contentType === 'nzb') {
|
||||
const xml = await this.readWhole(segment.data, file.message_ids)
|
||||
if (xml) result.nzb = (await NzbImportService.import(file, xml.toString('utf8'))) ?? undefined
|
||||
return result
|
||||
}
|
||||
const set = await Par2Service.read(segment.data, file.message_ids)
|
||||
if (set && binary) {
|
||||
result.par2 = await Par2Service.attach(binary, set)
|
||||
} else {
|
||||
const info = parseRarHeaders(segment.data)
|
||||
if (info) {
|
||||
result.archive = await ArchiveService.record(file, binary, info)
|
||||
if (hasNzbEntries(info)) result.archiveNzbs = await this.importArchiveNzbs(file, segment.data)
|
||||
}
|
||||
}
|
||||
if (binary) result.named = await NameService.resolve([binary.id])
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
/** NZBs that can be read from a single, unencrypted archive. */
|
||||
function hasNzbEntries(info: RarInfo): boolean {
|
||||
return !info.volume && !info.headersEncrypted && info.entries.some(isNzbEntry)
|
||||
}
|
||||
|
||||
function isNzbEntry(entry: RarEntry): boolean {
|
||||
return /\.nzb$/i.test(entry.name) && !entry.encrypted && !entry.directory
|
||||
}
|
||||
|
||||
/** Small posted files (NZBs) that are read completely. */
|
||||
const WHOLE_MAX_PARTS = 40
|
||||
const WHOLE_MAX_BYTES = 25 * 1024 * 1024
|
||||
|
||||
export default new ContentService()
|
||||
@@ -0,0 +1,264 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import QueueService from '#services/QueueService'
|
||||
import { type OverviewHeader } from '#services/IngestService'
|
||||
import { decodeSegment, inferTotalParts, parseYencMeta } from '#lib/yenc/index'
|
||||
import { sniffContentType } from '#lib/content/index'
|
||||
import { parseRarHeaders } from '#lib/rar/index'
|
||||
|
||||
/** Expanding re-reads the file's article range with XOVER; beyond this it is too costly. */
|
||||
const EXPAND_MAX_RANGE = 200_000
|
||||
/** A claimed row whose worker died is taken again after this long. */
|
||||
const RECLAIM_AFTER = '1 hour'
|
||||
|
||||
export interface DeferredArticle {
|
||||
articleNumber: number
|
||||
header: OverviewHeader
|
||||
}
|
||||
|
||||
/** File name from a deferred uploader's subject: `[i/M] "name" yEnc`, no part counter. */
|
||||
export function deferredName(subject: string | undefined): string | null {
|
||||
return /"([^"]+)"/.exec(subject ?? '')?.[1]?.trim() || null
|
||||
}
|
||||
|
||||
export function isDeferred(header: OverviewHeader): boolean {
|
||||
const id = header['message-id'] ?? ''
|
||||
return indexerConfig.deferBodyMessageIds.some((re) => re.test(id))
|
||||
}
|
||||
|
||||
/** A recorded range never spans more articles than this (one XOVER to expand it). */
|
||||
const RANGE_MAX_SPAN = 50_000
|
||||
|
||||
export function bodyJob(header: OverviewHeader, group: string, articleNumber: number | bigint) {
|
||||
return {
|
||||
name: 'process-body',
|
||||
data: { header, group, articleNumber: Number(articleNumber) },
|
||||
// By message-id: an article crossposted to several indexed groups is downloaded once.
|
||||
opts: { jobId: `body-${createHash('sha1').update(header['message-id'] ?? '').digest('hex')}` },
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deferred uploads (CONCEPT.md D28). Their articles skip the body queue and
|
||||
* are recorded here per file, from the headers alone. `worker:deferred`
|
||||
* later downloads one body per file for its size, part count and content
|
||||
* type, and sends small files (NZBs in RARs, par2 index files) to the body
|
||||
* queue completely.
|
||||
*/
|
||||
class DeferredService {
|
||||
/**
|
||||
* Records one group's deferred articles: per file when the subject names
|
||||
* the file, otherwise only as article ranges. Returns the number of files.
|
||||
*/
|
||||
async record(groupName: string, articles: DeferredArticle[]): Promise<number> {
|
||||
const named = articles.filter((a) => deferredName(a.header.subject) !== null)
|
||||
await this.recordRanges(groupName, articles.filter((a) => deferredName(a.header.subject) === null))
|
||||
return this.recordFiles(groupName, named)
|
||||
}
|
||||
|
||||
private async recordRanges(groupName: string, articles: DeferredArticle[]) {
|
||||
if (articles.length === 0) return
|
||||
const sorted = [...articles].sort((a, b) => Number(a.articleNumber) - Number(b.articleNumber))
|
||||
const now = new Date()
|
||||
const rows: Record<string, unknown>[] = []
|
||||
for (const article of sorted) {
|
||||
const n = Number(article.articleNumber)
|
||||
const bytes = Number.parseInt(article.header[':bytes'] ?? '0', 10) || 0
|
||||
const last = rows.at(-1)
|
||||
if (last && n - (last.first_article as number) <= RANGE_MAX_SPAN) {
|
||||
last.last_article = n
|
||||
;(last.article_count as number)++
|
||||
last.bytes = (last.bytes as number) + bytes
|
||||
} else {
|
||||
rows.push({ group_name: groupName, first_article: n, last_article: n, article_count: 1, bytes, created_at: now })
|
||||
}
|
||||
}
|
||||
await db.table('deferred_ranges').multiInsert(rows)
|
||||
}
|
||||
|
||||
private async recordFiles(groupName: string, articles: DeferredArticle[]): Promise<number> {
|
||||
const files = new Map<string, {
|
||||
name: string; subject: string; poster: string; first: number; firstId: string; last: number
|
||||
count: number; bytes: number; postedAt: Date | null
|
||||
}>()
|
||||
for (const { articleNumber, header } of articles) {
|
||||
const name = deferredName(header.subject)!
|
||||
const n = Number(articleNumber)
|
||||
const bytes = Number.parseInt(header[':bytes'] ?? '0', 10) || 0
|
||||
const date = header.date ? new Date(header.date) : null
|
||||
const postedAt = date && !Number.isNaN(date.getTime()) ? date : null
|
||||
const file = files.get(name)
|
||||
if (!file) {
|
||||
files.set(name, {
|
||||
name, subject: header.subject ?? '', poster: header.from ?? '', first: n,
|
||||
firstId: (header['message-id'] ?? '').replace(/^<|>$/g, ''), last: n, count: 1, bytes, postedAt,
|
||||
})
|
||||
continue
|
||||
}
|
||||
if (n < file.first) {
|
||||
file.first = n
|
||||
file.firstId = (header['message-id'] ?? '').replace(/^<|>$/g, '')
|
||||
}
|
||||
file.last = Math.max(file.last, n)
|
||||
file.count++
|
||||
file.bytes += bytes
|
||||
if (postedAt && (!file.postedAt || postedAt < file.postedAt)) file.postedAt = postedAt
|
||||
}
|
||||
if (files.size === 0) return 0
|
||||
|
||||
const now = new Date()
|
||||
const rows = [...files.values()].map((f) => ({
|
||||
file_key: createHash('sha1').update(`deferred\n${f.name}`).digest('hex'),
|
||||
name: f.name,
|
||||
subject: f.subject,
|
||||
poster: f.poster,
|
||||
group_name: groupName,
|
||||
first_article: f.first,
|
||||
first_message_id: f.firstId,
|
||||
last_article: f.last,
|
||||
article_count: f.count,
|
||||
bytes: f.bytes,
|
||||
posted_at: f.postedAt,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
}))
|
||||
// Counts and the article range only add up within the group the file was first seen in.
|
||||
await db.rawQuery(
|
||||
`INSERT INTO deferred_files AS d (${Object.keys(rows[0]).join(', ')})
|
||||
SELECT * FROM jsonb_to_recordset(?::jsonb) AS r(
|
||||
file_key varchar, name text, subject text, poster text, group_name text, first_article bigint,
|
||||
first_message_id text, last_article bigint, article_count int, bytes bigint, posted_at timestamptz,
|
||||
first_seen_at timestamptz, last_seen_at timestamptz)
|
||||
ON CONFLICT (file_key) DO UPDATE SET
|
||||
last_seen_at = EXCLUDED.last_seen_at,
|
||||
first_message_id = CASE WHEN d.group_name = EXCLUDED.group_name AND EXCLUDED.first_article < d.first_article
|
||||
THEN EXCLUDED.first_message_id ELSE d.first_message_id END,
|
||||
first_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN LEAST(d.first_article, EXCLUDED.first_article) ELSE d.first_article END,
|
||||
last_article = CASE WHEN d.group_name = EXCLUDED.group_name THEN GREATEST(d.last_article, EXCLUDED.last_article) ELSE d.last_article END,
|
||||
article_count = d.article_count + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.article_count ELSE 0 END,
|
||||
bytes = d.bytes + CASE WHEN d.group_name = EXCLUDED.group_name THEN EXCLUDED.bytes ELSE 0 END,
|
||||
posted_at = LEAST(d.posted_at, EXCLUDED.posted_at)`,
|
||||
[JSON.stringify(rows)]
|
||||
)
|
||||
return files.size
|
||||
}
|
||||
|
||||
/** Takes up to `limit` files that stopped growing; they are marked `processing`. */
|
||||
async claim(limit: number): Promise<any[]> {
|
||||
const settled = new Date(Date.now() - indexerConfig.deferredSettleMs)
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE deferred_files SET status = 'processing', processed_at = now()
|
||||
WHERE id IN (
|
||||
SELECT id FROM deferred_files
|
||||
WHERE (status = 'pending' AND last_seen_at < ?)
|
||||
OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
|
||||
ORDER BY id LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[settled, limit]
|
||||
)
|
||||
return rows
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues the bodies of up to `limit` recorded ranges, oldest first: the
|
||||
* articles are found again with XOVER. Returns ranges and articles queued.
|
||||
*/
|
||||
async expandRanges(limit: number): Promise<{ ranges: number; articles: number }> {
|
||||
const ranges = await db.from('deferred_ranges').whereNull('expanded_at').orderBy('id').limit(limit)
|
||||
let articles = 0
|
||||
for (const range of ranges) {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let overviews: [number, OverviewHeader][]
|
||||
try {
|
||||
await conn.group(range.group_name)
|
||||
overviews = (await conn.xover(String(range.first_article), String(range.last_article))).overviews
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
const jobs = overviews
|
||||
.filter(([, header]) => header && isDeferred(header) && deferredName(header.subject) === null)
|
||||
.map(([articleNumber, header]) => bodyJob(header, range.group_name, articleNumber))
|
||||
for (let i = 0; i < jobs.length; i += 5000) await QueueService.bodyQueue.addBulk(jobs.slice(i, i + 5000))
|
||||
await db.from('deferred_ranges').where('id', range.id).update({ expanded_at: new Date() })
|
||||
articles += jobs.length
|
||||
}
|
||||
return { ranges: ranges.length, articles }
|
||||
}
|
||||
|
||||
/** Reads one body of the file; expands small files into the body queue. Returns the new status. */
|
||||
async process(file: any): Promise<string> {
|
||||
const update: Record<string, unknown> = { processed_at: new Date(), error: null }
|
||||
try {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let body: Buffer
|
||||
try {
|
||||
body = await NntpService.fetchBody(conn, file.first_message_id)
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
|
||||
const meta = parseYencMeta(body)
|
||||
const totalParts = inferTotalParts(meta)
|
||||
Object.assign(update, {
|
||||
yenc_name: meta.begin.name,
|
||||
size: meta.begin.size,
|
||||
total_parts: totalParts,
|
||||
sampled_part: meta.begin.part ?? 1,
|
||||
})
|
||||
if ((meta.begin.part ?? 1) === 1) {
|
||||
const data = decodeSegment(body, { lenientSize: true, verifyCrc: false }).data
|
||||
update.content_type = sniffContentType(data)
|
||||
const rar = parseRarHeaders(data)
|
||||
if (rar) update.encrypted = rar.headersEncrypted || rar.entries.some((e) => e.encrypted)
|
||||
}
|
||||
|
||||
update.status = 'sampled'
|
||||
if (totalParts <= indexerConfig.deferredExpandMaxParts && update.encrypted !== true) {
|
||||
const queued = await this.expand(file)
|
||||
if (queued > 0) update.status = 'expanded'
|
||||
}
|
||||
} catch (error: any) {
|
||||
update.status = 'failed'
|
||||
update.error = String(error.message).slice(0, 500)
|
||||
}
|
||||
await db.from('deferred_files').where('id', file.id).update(update)
|
||||
return update.status as string
|
||||
}
|
||||
|
||||
/** Finds the file's articles again (XOVER over its range) and queues their bodies. */
|
||||
private async expand(file: any): Promise<number> {
|
||||
const first = Number(file.first_article)
|
||||
const last = Number(file.last_article)
|
||||
if (last - first > EXPAND_MAX_RANGE) return 0
|
||||
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
let overviews: [number, OverviewHeader][]
|
||||
try {
|
||||
await conn.group(file.group_name)
|
||||
overviews = (await conn.xover(String(first), String(last))).overviews
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
const jobs = overviews
|
||||
.filter(([, header]) => header && deferredName(header.subject) === file.name)
|
||||
.map(([articleNumber, header]) => bodyJob(header, file.group_name, articleNumber))
|
||||
if (jobs.length > 0) await QueueService.bodyQueue.addBulk(jobs)
|
||||
return jobs.length
|
||||
}
|
||||
}
|
||||
|
||||
export default new DeferredService()
|
||||
@@ -0,0 +1,115 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import QueueService from '#services/QueueService'
|
||||
|
||||
/** A failed range is retried after this long... */
|
||||
const REQUEUE_AFTER_MS = 5 * 60 * 1000
|
||||
/** ...at most this many times; after that it stays visible as a gap. */
|
||||
const MAX_REQUEUES = 5
|
||||
/** A queued range without a job in Redis (e.g. Redis was flushed) is re-queued after this long. */
|
||||
const LOST_AFTER_MS = 60 * 60 * 1000
|
||||
|
||||
interface Group {
|
||||
id: number
|
||||
name: string
|
||||
}
|
||||
|
||||
/**
|
||||
* Tracks the article ranges given to worker:fetch. A range only counts as
|
||||
* indexed once the worker reports it done, so failed or lost fetch jobs are
|
||||
* retried instead of silently skipped. See CONCEPT.md (D18).
|
||||
*/
|
||||
class FetchRangeService {
|
||||
async queue(group: Group, startId: bigint, endId: bigint) {
|
||||
const now = new Date()
|
||||
const [row] = await db
|
||||
.table('fetch_ranges')
|
||||
.insert({ group_id: group.id, start_id: startId.toString(), end_id: endId.toString(), created_at: now, updated_at: now })
|
||||
.returning(['id', 'requeues'])
|
||||
await this.addJob(group.name, Number(row.id), 0, startId, endId)
|
||||
}
|
||||
|
||||
private async addJob(groupName: string, rangeId: number, requeues: number, startId: bigint, endId: bigint) {
|
||||
await QueueService.nntpFetchQueue.add(
|
||||
'fetch-headers',
|
||||
{ groupName, startId: startId.toString(), endId: endId.toString(), rangeId },
|
||||
// The job id changes per attempt: BullMQ ignores an add() for an id it still knows.
|
||||
{ jobId: this.jobId(rangeId, requeues) }
|
||||
)
|
||||
}
|
||||
|
||||
private jobId(rangeId: number, requeues: number) {
|
||||
return `range-${rangeId}-${requeues}`
|
||||
}
|
||||
|
||||
async markDone(rangeId: number) {
|
||||
await db.from('fetch_ranges').where('id', rangeId).update({ status: 'done', error: null, updated_at: new Date() })
|
||||
}
|
||||
|
||||
async markFailed(rangeId: number, error: string) {
|
||||
await db.from('fetch_ranges').where('id', rangeId).update({ status: 'failed', error, updated_at: new Date() })
|
||||
}
|
||||
|
||||
/** Re-queues failed ranges, and queued ranges whose job no longer exists. */
|
||||
async requeue(group: Group): Promise<number> {
|
||||
const now = Date.now()
|
||||
const candidates = await db
|
||||
.from('fetch_ranges')
|
||||
.where('group_id', group.id)
|
||||
.where('requeues', '<', MAX_REQUEUES)
|
||||
.where((q) =>
|
||||
q
|
||||
.where((f) => f.where('status', 'failed').where('updated_at', '<', new Date(now - REQUEUE_AFTER_MS)))
|
||||
.orWhere((l) => l.where('status', 'queued').where('updated_at', '<', new Date(now - LOST_AFTER_MS)))
|
||||
)
|
||||
.orderBy('start_id')
|
||||
|
||||
let requeued = 0
|
||||
for (const range of candidates) {
|
||||
if (range.status === 'queued') {
|
||||
const job = await QueueService.nntpFetchQueue.getJob(this.jobId(Number(range.id), range.requeues))
|
||||
const state = job ? await job.getState() : 'missing'
|
||||
if (state !== 'missing' && state !== 'failed' && state !== 'unknown') continue
|
||||
}
|
||||
await db
|
||||
.from('fetch_ranges')
|
||||
.where('id', range.id)
|
||||
.update({ status: 'queued', requeues: range.requeues + 1, updated_at: new Date() })
|
||||
await this.addJob(group.name, Number(range.id), range.requeues + 1, BigInt(range.start_id), BigInt(range.end_id))
|
||||
requeued++
|
||||
logger.warn(
|
||||
`Re-queued ${group.name} ${range.start_id}-${range.end_id} (attempt ${range.requeues + 2}): ${range.error ?? 'job lost'}`
|
||||
)
|
||||
}
|
||||
return requeued
|
||||
}
|
||||
|
||||
/**
|
||||
* Moves `groups.last_indexed_id` up to just below the first range that is
|
||||
* not done yet, and forgets done ranges. Returns the new watermark.
|
||||
*/
|
||||
async advanceWatermark(group: Group): Promise<bigint> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE groups g SET last_indexed_id = greatest(g.last_indexed_id, coalesce(
|
||||
(SELECT min(start_id) - 1 FROM fetch_ranges WHERE group_id = g.id AND status <> 'done'),
|
||||
g.last_queued_id))
|
||||
WHERE g.id = ? RETURNING last_indexed_id`,
|
||||
[group.id]
|
||||
)
|
||||
await db.from('fetch_ranges').where('group_id', group.id).where('status', 'done').delete()
|
||||
return BigInt(rows[0].last_indexed_id)
|
||||
}
|
||||
|
||||
/** Ranges that failed for good (for monitoring). */
|
||||
async abandoned(group: Group) {
|
||||
return db
|
||||
.from('fetch_ranges')
|
||||
.where('group_id', group.id)
|
||||
.where('status', 'failed')
|
||||
.where('requeues', '>=', MAX_REQUEUES)
|
||||
.count('* as count')
|
||||
.first()
|
||||
}
|
||||
}
|
||||
|
||||
export default new FetchRangeService()
|
||||
@@ -0,0 +1,471 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import type { QueryClientContract } from '@adonisjs/lucid/types/database'
|
||||
import type { Knex } from 'knex'
|
||||
import QueueService from '#services/QueueService'
|
||||
import NameService from '#services/NameService'
|
||||
import { parseXrefGroups } from '#lib/nntp/xref'
|
||||
import { fileKey, parseSubject, weakFileKey } from '#lib/subject/index'
|
||||
import { releaseIdentity, type ReleaseIdentity } from '#lib/release/index'
|
||||
import { inferTotalParts, type YencMeta } from '#lib/yenc/index'
|
||||
|
||||
/** One segment, with the identity of the file it belongs to already resolved. */
|
||||
export interface IngestSegment {
|
||||
fileKey: string
|
||||
poster: string
|
||||
/** Subject without segment counter. */
|
||||
subject: string
|
||||
name: string
|
||||
totalParts: number
|
||||
number: number
|
||||
/** Without angle brackets. */
|
||||
messageId: string
|
||||
bytes: number
|
||||
articleNumber?: bigint | number
|
||||
postedAt?: Date
|
||||
groups: string[]
|
||||
/**
|
||||
* The identity comes from the yEnc name only (obfuscated posts). Such names
|
||||
* can be shared by several equal-sized files of one post, so a second
|
||||
* article for the same part number marks the binary as conflicted.
|
||||
*/
|
||||
weakIdentity?: boolean
|
||||
/** Release (download) this file belongs to, when it can be determined. */
|
||||
release?: ReleaseIdentity | null
|
||||
/** Exact file size from =ybegin (body-derived segments only). */
|
||||
size?: number
|
||||
/** yEnc CRCs (body-derived segments only), to verify split volumes. */
|
||||
pcrc32?: string
|
||||
partSize?: number
|
||||
fileCrc32?: string
|
||||
/** Part 1 only: MD5 of the first 16 KiB of the file, as par2 records it. */
|
||||
md5_16k?: string
|
||||
}
|
||||
|
||||
export interface CompletedBinary {
|
||||
id: number
|
||||
fileKey: string
|
||||
}
|
||||
|
||||
const BINARY_CHUNK = 1000
|
||||
/** last_seen_at is only rewritten when it is older than this (it drives GC at 48 h). */
|
||||
const TOUCH_MS = 10 * 60 * 1000
|
||||
|
||||
type Merge = 'least' | 'coalesce' | 'touch'
|
||||
|
||||
/**
|
||||
* Inserts rows keyed by `key` and returns their ids. Existing rows are not
|
||||
* locked unless something in them changes (`least`: an earlier value,
|
||||
* `coalesce`: a value where there was none, `touch`: a timestamp more than
|
||||
* TOUCH_MS newer). Parallel fetch jobs see mostly the same rows, and
|
||||
* `INSERT … ON CONFLICT DO UPDATE` made them wait on each other for every one.
|
||||
*/
|
||||
async function upsertKeyed(
|
||||
knex: Knex,
|
||||
table: 'binaries' | 'releases',
|
||||
key: string,
|
||||
rows: Record<string, unknown>[],
|
||||
merge: Record<string, [Merge, string]>
|
||||
): Promise<Map<string, number>> {
|
||||
const ids = new Map<string, number>()
|
||||
if (rows.length === 0) return ids
|
||||
const inserted: Record<string, any>[] = await knex(table).insert(rows).onConflict(key).ignore().returning(['id', key])
|
||||
for (const row of inserted) ids.set(row[key], Number(row.id))
|
||||
|
||||
const incoming = new Map(rows.filter((r) => !ids.has(r[key] as string)).map((r) => [r[key] as string, r]))
|
||||
if (incoming.size === 0) return ids
|
||||
const columns = Object.keys(merge)
|
||||
const existing: Record<string, any>[] = await knex(table).whereIn(key, [...incoming.keys()]).select('id', key, ...columns)
|
||||
|
||||
const changes: Record<string, unknown>[] = []
|
||||
for (const row of existing) {
|
||||
ids.set(row[key], Number(row.id))
|
||||
const next = incoming.get(row[key])!
|
||||
const changed = columns.some((column) => {
|
||||
const [kind] = merge[column]
|
||||
const current = row[column]
|
||||
const value = next[column]
|
||||
if (value === null || value === undefined) return false
|
||||
if (kind === 'coalesce') return current === null
|
||||
if (kind === 'least') return current === null || value < current
|
||||
return current === null || +(value as Date) - +current > TOUCH_MS
|
||||
})
|
||||
if (changed) changes.push({ id: Number(row.id), ...Object.fromEntries(columns.map((c) => [c, next[c] ?? null])) })
|
||||
}
|
||||
if (changes.length === 0) return ids
|
||||
|
||||
const set = columns
|
||||
.map((c) => {
|
||||
const [kind] = merge[c]
|
||||
const fn = kind === 'least' ? 'least' : kind === 'touch' ? 'greatest' : 'coalesce'
|
||||
return `${c} = ${fn}(t.${c}, v.${c})`
|
||||
})
|
||||
.join(', ')
|
||||
const types = columns.map((c) => `${c} ${merge[c][1]}`).join(', ')
|
||||
changes.sort((a, b) => (a.id as number) - (b.id as number))
|
||||
for (let attempt = 1; ; attempt++) {
|
||||
try {
|
||||
await knex.raw(
|
||||
`UPDATE ${table} t SET ${set}
|
||||
FROM jsonb_to_recordset(?::jsonb) AS v(id bigint, ${types})
|
||||
WHERE t.id = v.id`,
|
||||
[JSON.stringify(changes)]
|
||||
)
|
||||
return ids
|
||||
} catch (error: any) {
|
||||
// Rare now, but two jobs can still update overlapping rows in a different order.
|
||||
if (error.code !== '40P01' || attempt >= 3) throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
/** Size-only matching below this is too ambiguous (nfo, sfv, small volumes). */
|
||||
export const PAR2_SIZE_ONLY_MIN = 1024 * 1024
|
||||
/** A par2 set and its files are posted together; don't match across days. */
|
||||
export const PAR2_MATCH_WINDOW_SECONDS = 12 * 60 * 60
|
||||
const PART_CHUNK = 2000
|
||||
|
||||
function chunks<T>(items: T[], size: number): T[][] {
|
||||
const result: T[][] = []
|
||||
for (let i = 0; i < items.length; i += size) result.push(items.slice(i, i + size))
|
||||
return result
|
||||
}
|
||||
|
||||
function validDate(date?: Date) {
|
||||
return date && !Number.isNaN(date.getTime()) ? date : null
|
||||
}
|
||||
|
||||
/** An XOVER overview entry as returned by nntp-js. */
|
||||
export type OverviewHeader = Record<string, string>
|
||||
|
||||
function common(header: OverviewHeader, groupName: string, articleNumber?: bigint | number) {
|
||||
const xrefGroups = parseXrefGroups(header.xref)
|
||||
return {
|
||||
poster: header.from ?? '',
|
||||
messageId: (header['message-id'] ?? '').replace(/^<|>$/g, ''),
|
||||
bytes: Number.parseInt(header[':bytes'] ?? '0', 10) || 0,
|
||||
articleNumber,
|
||||
postedAt: header.date ? new Date(header.date) : undefined,
|
||||
groups: xrefGroups.length > 0 ? xrefGroups : [groupName],
|
||||
}
|
||||
}
|
||||
|
||||
/** Builds a segment from the subject line; null when the subject is obfuscated. */
|
||||
export function segmentFromHeader(
|
||||
header: OverviewHeader,
|
||||
groupName: string,
|
||||
articleNumber?: bigint | number
|
||||
): IngestSegment | null {
|
||||
const parsed = header.subject ? parseSubject(header.subject) : null
|
||||
if (!parsed || !header['message-id']) return null
|
||||
const base = common(header, groupName, articleNumber)
|
||||
return {
|
||||
...base,
|
||||
fileKey: fileKey(base.poster, parsed.base, parsed.totalSegments),
|
||||
subject: parsed.base,
|
||||
name: parsed.name,
|
||||
totalParts: parsed.totalSegments,
|
||||
number: parsed.segment,
|
||||
release: releaseIdentity({ poster: base.poster, subject: parsed.base, name: parsed.name, weak: false }),
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds a segment for an obfuscated post from the yEnc header in its body.
|
||||
* Subject and poster can be random per article, so the identity comes from
|
||||
* the yEnc header only (see weakFileKey).
|
||||
*/
|
||||
export function segmentFromYenc(
|
||||
header: OverviewHeader,
|
||||
groupName: string,
|
||||
meta: Pick<YencMeta, 'begin' | 'part'> & Partial<Pick<YencMeta, 'end'>>,
|
||||
articleNumber?: bigint | number
|
||||
): IngestSegment | null {
|
||||
const base = common(header, groupName, articleNumber)
|
||||
const totalParts = inferTotalParts(meta)
|
||||
const number = meta.begin.part ?? 1
|
||||
if (!base.messageId || number < 1 || number > totalParts) return null
|
||||
const subject = `"${meta.begin.name}" yEnc`
|
||||
return {
|
||||
...base,
|
||||
fileKey: weakFileKey(meta.begin.name, meta.begin.size, totalParts),
|
||||
subject,
|
||||
name: meta.begin.name,
|
||||
totalParts,
|
||||
number,
|
||||
weakIdentity: true,
|
||||
release: releaseIdentity({ poster: base.poster, subject, name: meta.begin.name, weak: true }),
|
||||
size: meta.begin.size,
|
||||
// Single-part posts only carry crc32, which is then also the part's CRC.
|
||||
pcrc32: meta.end?.pcrc32 ?? (meta.part ? undefined : meta.end?.crc32),
|
||||
partSize: meta.part ? meta.part.end - meta.part.begin + 1 : meta.begin.size,
|
||||
fileCrc32: meta.end?.crc32,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stores segments in `binaries`/`parts` and queues every binary that became
|
||||
* complete exactly once. Idempotent: ingesting the same segments again is a
|
||||
* no-op. See CONCEPT.md (D12).
|
||||
*/
|
||||
class IngestService {
|
||||
async ingest(segments: IngestSegment[]): Promise<CompletedBinary[]> {
|
||||
if (segments.length === 0) return []
|
||||
|
||||
// Short statements, no enclosing transaction: every step is idempotent, and
|
||||
// row locks held for a whole batch made parallel fetch jobs wait on each other.
|
||||
const binaryIds = await (async () => {
|
||||
const knex = db.connection().getWriteClient()
|
||||
const now = new Date()
|
||||
const releaseIds = await this.upsertReleases(knex, segments, now)
|
||||
|
||||
// One row per file_key (ON CONFLICT cannot touch a row twice per statement),
|
||||
// sorted so concurrent workers lock rows in the same order and cannot deadlock.
|
||||
const binaries = new Map<string, Record<string, unknown>>()
|
||||
for (const segment of segments) {
|
||||
const postedAt = validDate(segment.postedAt)
|
||||
const existing = binaries.get(segment.fileKey)
|
||||
if (existing) {
|
||||
if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
|
||||
existing.posted_at = postedAt
|
||||
}
|
||||
continue
|
||||
}
|
||||
binaries.set(segment.fileKey, {
|
||||
file_key: segment.fileKey,
|
||||
poster: segment.poster,
|
||||
subject: segment.subject,
|
||||
name: segment.name,
|
||||
total_parts: segment.totalParts,
|
||||
groups: JSON.stringify(segment.groups),
|
||||
posted_at: postedAt,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
release_id: segment.release ? releaseIds.get(segment.release.key) : null,
|
||||
size: segment.size ?? null,
|
||||
md5_16k: segment.md5_16k ?? null,
|
||||
})
|
||||
}
|
||||
const binaryRows = [...binaries.values()].sort((a, b) =>
|
||||
(a.file_key as string).localeCompare(b.file_key as string)
|
||||
)
|
||||
|
||||
const idsByKey = new Map<string, number>()
|
||||
for (const chunk of chunks(binaryRows, BINARY_CHUNK)) {
|
||||
const ids = await upsertKeyed(knex, 'binaries', 'file_key', chunk, {
|
||||
last_seen_at: ['touch', 'timestamptz'],
|
||||
posted_at: ['least', 'timestamptz'],
|
||||
release_id: ['coalesce', 'bigint'],
|
||||
md5_16k: ['coalesce', 'varchar'],
|
||||
})
|
||||
for (const [key, id] of ids) idsByKey.set(key, id)
|
||||
}
|
||||
|
||||
const partRows = segments
|
||||
.map((segment) => ({
|
||||
binary_id: idsByKey.get(segment.fileKey)!,
|
||||
number: segment.number,
|
||||
message_id: segment.messageId,
|
||||
bytes: segment.bytes,
|
||||
article_number: segment.articleNumber?.toString() ?? null,
|
||||
pcrc32: segment.pcrc32 ?? null,
|
||||
part_size: segment.partSize ?? null,
|
||||
file_crc32: segment.fileCrc32 ?? null,
|
||||
md5_16k: segment.md5_16k ?? null,
|
||||
}))
|
||||
const weakRows = partRows.filter((_, i) => segments[i].weakIdentity)
|
||||
|
||||
partRows.sort((a, b) => a.binary_id - b.binary_id || a.number - b.number)
|
||||
for (const chunk of chunks(partRows, PART_CHUNK)) {
|
||||
await knex('parts').insert(chunk).onConflict(['binary_id', 'number']).ignore()
|
||||
}
|
||||
|
||||
if (weakRows.length > 0) await this.flagConflicts(db.connection(), weakRows)
|
||||
|
||||
return [...idsByKey.values()]
|
||||
})()
|
||||
|
||||
// par2 naming needs a size, which only body-derived segments have; for
|
||||
// header batches these queries were most of Postgres' work.
|
||||
const sized = [...new Set(segments.filter((s) => s.size != null).map((s) => s.fileKey))]
|
||||
if (sized.length > 0) {
|
||||
const sizedIds = (await db.from('binaries').whereIn('file_key', sized).select('id')).map((r) => Number(r.id))
|
||||
await this.matchPar2Releases(sizedIds)
|
||||
await NameService.resolve(sizedIds)
|
||||
}
|
||||
|
||||
// After commit, so the last writer of a binary always sees all of its parts.
|
||||
const completed = await this.claimCompleted(binaryIds)
|
||||
await this.enqueue(completed)
|
||||
return completed
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates or touches the releases referenced by `segments` and returns
|
||||
* their ids by release key. Also used by `release:rebuild`.
|
||||
*/
|
||||
async upsertReleases(
|
||||
knex: Knex,
|
||||
segments: Pick<IngestSegment, 'release' | 'poster' | 'groups' | 'postedAt' | 'weakIdentity'>[],
|
||||
now = new Date()
|
||||
): Promise<Map<string, number>> {
|
||||
const releases = new Map<string, Record<string, unknown>>()
|
||||
for (const segment of segments) {
|
||||
const release = segment.release
|
||||
if (!release) continue
|
||||
const postedAt = validDate(segment.postedAt)
|
||||
const existing = releases.get(release.key)
|
||||
if (existing) {
|
||||
if (postedAt && (!existing.posted_at || postedAt < (existing.posted_at as Date))) {
|
||||
existing.posted_at = postedAt
|
||||
}
|
||||
// Not every file carries the [i/M] counter (e.g. a par2 index): take it from any that does.
|
||||
if (existing.file_total === null && release.fileTotal) existing.file_total = release.fileTotal
|
||||
continue
|
||||
}
|
||||
releases.set(release.key, {
|
||||
release_key: release.key,
|
||||
name: release.name,
|
||||
// Obfuscating posters randomise the poster, so it says nothing about the release.
|
||||
poster: segment.weakIdentity ? null : segment.poster,
|
||||
file_total: release.fileTotal ?? null,
|
||||
source: release.source,
|
||||
groups: JSON.stringify(segment.groups),
|
||||
posted_at: postedAt,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
})
|
||||
}
|
||||
|
||||
const ids = new Map<string, number>()
|
||||
const rows = [...releases.values()].sort((a, b) =>
|
||||
(a.release_key as string).localeCompare(b.release_key as string)
|
||||
)
|
||||
for (const chunk of chunks(rows, BINARY_CHUNK)) {
|
||||
const result = await upsertKeyed(knex, 'releases', 'release_key', chunk, {
|
||||
last_seen_at: ['touch', 'timestamptz'],
|
||||
posted_at: ['least', 'timestamptz'],
|
||||
file_total: ['coalesce', 'integer'],
|
||||
source: ['coalesce', 'varchar'],
|
||||
})
|
||||
for (const [key, id] of result) ids.set(key, id)
|
||||
}
|
||||
return ids
|
||||
}
|
||||
|
||||
/**
|
||||
* Assigns body-derived binaries without a release and without a
|
||||
* fingerprint to a known par2 set by size: only when that size is listed
|
||||
* once in the set, the file is large, and exactly one set nearby lists it.
|
||||
* Binaries with a fingerprint are matched exactly by NameService instead.
|
||||
*/
|
||||
async matchPar2Releases(binaryIds: number[]) {
|
||||
if (binaryIds.length === 0) return
|
||||
await db.rawQuery(
|
||||
`UPDATE binaries b SET release_id = m.release_id
|
||||
FROM (
|
||||
SELECT b2.id, min(rf.release_id) AS release_id
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.size = b2.size
|
||||
JOIN releases r ON r.id = rf.release_id
|
||||
WHERE b2.id = ANY(?) AND b2.release_id IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
|
||||
AND abs(extract(epoch FROM (r.posted_at - b2.posted_at))) < ?
|
||||
AND (SELECT count(*) FROM release_files x WHERE x.release_id = rf.release_id AND x.size = rf.size) = 1
|
||||
GROUP BY b2.id
|
||||
HAVING count(DISTINCT rf.release_id) = 1
|
||||
) m
|
||||
WHERE b.id = m.id`,
|
||||
[binaryIds, PAR2_SIZE_ONLY_MIN, PAR2_MATCH_WINDOW_SECONDS]
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Flags binaries where a stored part has a different message-id and a
|
||||
* different part CRC than the incoming one: two files were merged under
|
||||
* one identity. Identical reposts (same CRC) are ignored.
|
||||
*/
|
||||
private async flagConflicts(
|
||||
trx: QueryClientContract,
|
||||
rows: {
|
||||
binary_id: number
|
||||
number: number
|
||||
message_id: string
|
||||
bytes: number
|
||||
article_number: string | null
|
||||
pcrc32: string | null
|
||||
part_size: number | null
|
||||
file_crc32: string | null
|
||||
md5_16k: string | null
|
||||
}[]
|
||||
) {
|
||||
const bindings = [
|
||||
rows.map((r) => r.binary_id),
|
||||
rows.map((r) => r.number),
|
||||
rows.map((r) => r.message_id),
|
||||
rows.map((r) => r.bytes),
|
||||
rows.map((r) => r.article_number),
|
||||
rows.map((r) => r.pcrc32),
|
||||
rows.map((r) => r.part_size),
|
||||
rows.map((r) => r.file_crc32),
|
||||
rows.map((r) => r.md5_16k),
|
||||
]
|
||||
// Keep the extra articles: SplitService separates the volumes from them.
|
||||
const conflicts = await trx.rawQuery(
|
||||
`INSERT INTO part_conflicts (binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
|
||||
SELECT v.binary_id, v.number, v.message_id, v.bytes, v.article_number, v.pcrc32, v.part_size, v.file_crc32, v.md5_16k
|
||||
FROM unnest(?::bigint[], ?::int[], ?::text[], ?::int[], ?::bigint[], ?::text[], ?::int[], ?::text[], ?::text[])
|
||||
AS v(binary_id, number, message_id, bytes, article_number, pcrc32, part_size, file_crc32, md5_16k)
|
||||
JOIN parts p ON p.binary_id = v.binary_id AND p.number = v.number
|
||||
WHERE p.message_id <> v.message_id
|
||||
-- Same part CRC: the same data posted again (a repost), not another file.
|
||||
AND (p.pcrc32 IS NULL OR v.pcrc32 IS NULL OR p.pcrc32 <> v.pcrc32)
|
||||
ON CONFLICT DO NOTHING
|
||||
RETURNING binary_id`,
|
||||
bindings as any
|
||||
)
|
||||
const binaryIds: number[] = [
|
||||
...new Set<number>(conflicts.rows.map((r: { binary_id: string }) => Number(r.binary_id))),
|
||||
]
|
||||
if (binaryIds.length === 0) return
|
||||
const result = await trx.rawQuery(
|
||||
`UPDATE binaries SET conflicted_at = now()
|
||||
WHERE conflicted_at IS NULL AND id = ANY(?) RETURNING name`,
|
||||
[binaryIds]
|
||||
)
|
||||
for (const row of result.rows) {
|
||||
logger.warn(`Binary "${row.name}" has conflicting parts; it will be split into volumes.`)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Marks complete binaries among `binaryIds` as claimed. The single UPDATE
|
||||
* locks each row, so concurrent callers never claim the same binary twice.
|
||||
*/
|
||||
async claimCompleted(binaryIds: number[]): Promise<CompletedBinary[]> {
|
||||
if (binaryIds.length === 0) return []
|
||||
const result = await db.rawQuery(
|
||||
`UPDATE binaries b SET completed_at = now()
|
||||
WHERE b.id = ANY(?) AND b.completed_at IS NULL AND b.conflicted_at IS NULL
|
||||
AND (SELECT count(*) FROM parts p WHERE p.binary_id = b.id) >= b.total_parts
|
||||
RETURNING b.id, b.file_key`,
|
||||
[binaryIds]
|
||||
)
|
||||
return result.rows.map((row: { id: string; file_key: string }) => ({
|
||||
id: Number(row.id),
|
||||
fileKey: row.file_key,
|
||||
}))
|
||||
}
|
||||
|
||||
async enqueue(completed: CompletedBinary[]) {
|
||||
if (completed.length === 0) return
|
||||
// jobId = file_key: BullMQ ignores an add() for an id that already exists.
|
||||
await QueueService.fileQueue.addBulk(
|
||||
completed.map(({ id, fileKey }) => ({
|
||||
name: 'process-file',
|
||||
data: { binaryId: id },
|
||||
opts: { jobId: fileKey },
|
||||
}))
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
export default new IngestService()
|
||||
@@ -0,0 +1,128 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
|
||||
/**
|
||||
* A release that is nothing more than one randomly named file (its own
|
||||
* filename stem). Such a file moves into the par2 set that names it exactly;
|
||||
* real groupings (par2, RAR, NZB, spot, subject, post id) are left alone.
|
||||
*/
|
||||
export const WEAK_RELEASE_SQL = (column: string) =>
|
||||
`EXISTS (SELECT 1 FROM releases wr WHERE wr.id = ${column} AND wr.source IN ('stem', 'weak-stem')
|
||||
AND wr.file_count <= 1 AND (wr.name !~ '[._ -]' OR NOT is_readable_name(wr.name)))`
|
||||
|
||||
/**
|
||||
* Whether `candidate` (e.g. from a par2 listing) should replace `current`:
|
||||
* the current name is random, or a single token without separators
|
||||
* (mOzyyxCyYUj, F8lv3w5agMDI3; too short for looksRandom), while the
|
||||
* candidate reads like a release name.
|
||||
*/
|
||||
export function isBetterName(current: string, candidate: string): boolean {
|
||||
if (looksRandom(candidate) || !/[._ -]/.test(candidate)) return false
|
||||
return looksRandom(current) || !/[._ -]/.test(current)
|
||||
}
|
||||
|
||||
/** Smaller files (nfo, sfv) collide too easily on size alone. */
|
||||
const MIN_SIZE_ONLY = 64 * 1024
|
||||
|
||||
/**
|
||||
* Gives binaries (and their files) the real filename that a par2 set lists
|
||||
* for them. See CONCEPT.md (D21).
|
||||
* - Exact: size and MD5 of the first 16 KiB both match a par2 entry. This
|
||||
* also puts the binary in that par2 set's release.
|
||||
* - Size only, when the fingerprint is unknown: within the binary's own
|
||||
* release, only when exactly one listed file has that size.
|
||||
*/
|
||||
class NameService {
|
||||
async resolve(binaryIds: number[]): Promise<number> {
|
||||
if (binaryIds.length === 0) return 0
|
||||
const exact = await db.rawQuery(
|
||||
`UPDATE binaries b SET real_name = m.name,
|
||||
release_id = CASE WHEN b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')} THEN m.release_id ELSE b.release_id END
|
||||
FROM (
|
||||
SELECT DISTINCT ON (b2.id) b2.id, rf.name, rf.release_id
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.size = b2.size AND rf.md5_16k = b2.md5_16k
|
||||
WHERE b2.id = ANY(?)
|
||||
ORDER BY b2.id, rf.release_id DESC
|
||||
) m
|
||||
WHERE b.id = m.id AND (b.real_name IS DISTINCT FROM m.name OR b.release_id IS NULL
|
||||
OR (b.release_id <> m.release_id AND ${WEAK_RELEASE_SQL('b.release_id')}))`,
|
||||
[binaryIds]
|
||||
)
|
||||
const bySize = await db.rawQuery(
|
||||
`UPDATE binaries b SET real_name = m.name
|
||||
FROM (
|
||||
SELECT b2.id, min(rf.name) AS name
|
||||
FROM binaries b2
|
||||
JOIN release_files rf ON rf.release_id = b2.release_id AND rf.size = b2.size
|
||||
WHERE b2.id = ANY(?) AND b2.real_name IS NULL AND b2.md5_16k IS NULL AND b2.size >= ?
|
||||
GROUP BY b2.id
|
||||
HAVING count(DISTINCT rf.name) = 1
|
||||
) m
|
||||
WHERE b.id = m.id`,
|
||||
[binaryIds, MIN_SIZE_ONLY]
|
||||
)
|
||||
await db.rawQuery(
|
||||
`UPDATE files f SET real_name = b.real_name,
|
||||
release_id = CASE WHEN f.release_id IS NULL OR ${WEAK_RELEASE_SQL('f.release_id')} THEN coalesce(b.release_id, f.release_id) ELSE f.release_id END
|
||||
FROM binaries b
|
||||
WHERE b.id = ANY(?) AND b.file_id = f.id
|
||||
AND (f.real_name IS DISTINCT FROM b.real_name OR (f.release_id IS NULL AND b.release_id IS NOT NULL)
|
||||
OR (f.release_id <> b.release_id AND ${WEAK_RELEASE_SQL('f.release_id')}))`,
|
||||
[binaryIds]
|
||||
)
|
||||
return (exact.rowCount ?? 0) + (bySize.rowCount ?? 0)
|
||||
}
|
||||
|
||||
/**
|
||||
* When a par2 set arrives: the binaries it can name, i.e. those matching
|
||||
* one of its entries exactly (size + first-16-KiB MD5) and those in its
|
||||
* release. Selecting every binary of the listed sizes instead picked up
|
||||
* hundreds of thousands of equal-sized volumes per set (LOG.md 2026-09-28 (18)).
|
||||
*/
|
||||
async resolveForRelease(releaseId: number): Promise<number> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT b.id FROM release_files rf
|
||||
JOIN binaries b ON b.md5_16k = rf.md5_16k AND b.size = rf.size
|
||||
WHERE rf.release_id = ?
|
||||
UNION
|
||||
SELECT id FROM binaries WHERE release_id = ? AND size IS NOT NULL`,
|
||||
[releaseId, releaseId]
|
||||
)
|
||||
return this.resolve(rows.map((r: { id: string }) => Number(r.id)))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Repair for data from before files moved into the par2 set that names them
|
||||
* (used by `release:rename`): regroups exact matches out of random-named
|
||||
* single-file releases, then renames par2 releases that still carry a
|
||||
* random name after their listing. No downloads.
|
||||
*/
|
||||
export async function regroupAndRenameFromPar2(): Promise<{ regrouped: number; renamed: number }> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`SELECT DISTINCT b.id FROM binaries b
|
||||
JOIN release_files rf ON rf.md5_16k = b.md5_16k AND rf.size = b.size
|
||||
WHERE b.release_id IS DISTINCT FROM rf.release_id
|
||||
AND (b.release_id IS NULL OR ${WEAK_RELEASE_SQL('b.release_id')})`
|
||||
)
|
||||
const ids = rows.map((r: { id: string }) => Number(r.id))
|
||||
for (let i = 0; i < ids.length; i += 1000) await nameService.resolve(ids.slice(i, i + 1000))
|
||||
|
||||
const { par2ReleaseName } = await import('#lib/release/index')
|
||||
const listings = await db.rawQuery(
|
||||
`SELECT r.id, r.name, array_agg(rf.name) AS names FROM releases r
|
||||
JOIN release_files rf ON rf.release_id = r.id GROUP BY r.id, r.name`
|
||||
)
|
||||
let renamed = 0
|
||||
for (const release of listings.rows) {
|
||||
const better = par2ReleaseName(release.names)
|
||||
if (!better || better === release.name || !isBetterName(release.name, better)) continue
|
||||
await db.from('releases').where('id', release.id).update({ name: better })
|
||||
renamed++
|
||||
}
|
||||
return { regrouped: ids.length, renamed }
|
||||
}
|
||||
|
||||
const nameService = new NameService()
|
||||
export default nameService
|
||||
+68
-21
@@ -1,5 +1,6 @@
|
||||
import nntpConfig from '#config/nntp'
|
||||
import { NNTP } from "nntp-js";
|
||||
import { parseGroupResponse, type GroupInfo } from '#lib/nntp/group'
|
||||
|
||||
class NntpService {
|
||||
private readonly poolSize: number;
|
||||
@@ -14,15 +15,20 @@ class NntpService {
|
||||
this.idleConnections = [];
|
||||
this.waiters = [];
|
||||
this.createdCount = 0;
|
||||
console.log(`NNTP Pool initialized with size ${this.poolSize}`)
|
||||
}
|
||||
|
||||
private async _createConnection() {
|
||||
// This connection logic is based on the older, working pool implementation.
|
||||
const conn = new NNTP(nntpConfig.host, nntpConfig.port);
|
||||
await conn.connect();
|
||||
if (nntpConfig.user) {
|
||||
await conn.login(nntpConfig.user, nntpConfig.password?.release());
|
||||
try {
|
||||
await conn.connect();
|
||||
if (nntpConfig.user) {
|
||||
await conn.login(nntpConfig.user, nntpConfig.password?.release());
|
||||
}
|
||||
} catch (error) {
|
||||
// Free the slot, otherwise a failed connect shrinks the pool for good.
|
||||
this.createdCount--;
|
||||
this._destroy(conn);
|
||||
throw error;
|
||||
}
|
||||
this.allConnections.add(conn);
|
||||
return conn;
|
||||
@@ -30,38 +36,79 @@ class NntpService {
|
||||
|
||||
public async acquire() {
|
||||
if (this.idleConnections.length > 0) {
|
||||
console.log('Reusing existing connection from pool.');
|
||||
return this.idleConnections.pop();
|
||||
}
|
||||
|
||||
if (this.createdCount < this.poolSize) {
|
||||
this.createdCount++;
|
||||
console.log(`Creating new connection (${this.createdCount}/${this.poolSize}).`);
|
||||
console.log(`Creating new connection (${this.createdCount}/${this.poolSize}).`);
|
||||
return this._createConnection();
|
||||
}
|
||||
|
||||
console.log(`Pool maxed out at ${this.poolSize}. Waiting for a connection to become available.`);
|
||||
return new Promise(resolve => this.waiters.push(resolve));
|
||||
}
|
||||
|
||||
public release(conn: any) {
|
||||
if (this.waiters.length > 0) {
|
||||
console.log('Releasing connection directly to a waiting task.');
|
||||
const resolve = this.waiters.shift();
|
||||
if(resolve) resolve(conn);
|
||||
/**
|
||||
* Returns a connection to the pool. Pass `failed` when a command on it
|
||||
* threw: the connection may be dead (e.g. closed by the server while idle),
|
||||
* so it is discarded instead of being handed to the next job.
|
||||
*/
|
||||
public release(conn: any, failed = false) {
|
||||
if (failed) {
|
||||
this.discard(conn);
|
||||
return;
|
||||
}
|
||||
const resolve = this.waiters.shift();
|
||||
if (resolve) {
|
||||
resolve(conn);
|
||||
} else {
|
||||
console.log('Returning connection to the idle pool.');
|
||||
this.idleConnections.push(conn);
|
||||
}
|
||||
}
|
||||
|
||||
public async shutdown() {
|
||||
console.log('Shutting down all connections in the pool.');
|
||||
const shutdownPromises: Promise<any>[] = [];
|
||||
for (const conn of this.allConnections) {
|
||||
shutdownPromises.push(conn.quit());
|
||||
/** Closes a broken connection and frees its slot for a new one. */
|
||||
public discard(conn: any) {
|
||||
this.allConnections.delete(conn);
|
||||
this.createdCount--;
|
||||
this._destroy(conn);
|
||||
|
||||
// Someone is waiting for a slot: open a fresh connection for them.
|
||||
const resolve = this.waiters.shift();
|
||||
if (resolve) {
|
||||
this.createdCount++;
|
||||
this._createConnection().then(resolve, () => this.waiters.unshift(resolve));
|
||||
}
|
||||
await Promise.all(shutdownPromises);
|
||||
}
|
||||
|
||||
private _destroy(conn: any) {
|
||||
try {
|
||||
conn.sock?.destroy();
|
||||
} catch {
|
||||
// already closed
|
||||
}
|
||||
}
|
||||
|
||||
/** Selects a group and returns its correctly parsed article range. */
|
||||
public async selectGroup(conn: any, name: string): Promise<GroupInfo> {
|
||||
const { resp } = await conn.group(name)
|
||||
return parseGroupResponse(resp)
|
||||
}
|
||||
|
||||
/** Fetches a raw (still dot-stuffed) article body as a Buffer. */
|
||||
public async fetchBody(conn: any, messageId: string): Promise<Buffer> {
|
||||
const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
|
||||
const { data } = await conn.body(id)
|
||||
return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
|
||||
}
|
||||
|
||||
public async shutdown() {
|
||||
console.log('Shutting down all connections in the pool.');
|
||||
// A connection the server already closed may never answer QUIT.
|
||||
const quit = (conn: any) =>
|
||||
Promise.race([conn.quit(), new Promise((resolve) => setTimeout(resolve, 2000).unref())])
|
||||
.catch(() => {})
|
||||
.finally(() => this._destroy(conn));
|
||||
await Promise.all([...this.allConnections].map(quit));
|
||||
this.allConnections.clear();
|
||||
this.idleConnections.length = 0;
|
||||
this.waiters.length = 0;
|
||||
@@ -69,4 +116,4 @@ class NntpService {
|
||||
}
|
||||
}
|
||||
|
||||
export default new NntpService();
|
||||
export default new NntpService(nntpConfig.poolSize);
|
||||
@@ -0,0 +1,168 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import IngestService from '#services/IngestService'
|
||||
import { type ReleaseIdentity } from '#lib/release/index'
|
||||
import { parseNzb, type ParsedNzb } from '#lib/nzb/index'
|
||||
import { parseSubject } from '#lib/subject/index'
|
||||
import { fileStem } from '#lib/release/index'
|
||||
import { looksRandom } from '#services/ArchiveService'
|
||||
|
||||
export interface NzbImportResult {
|
||||
releaseId: number
|
||||
releaseName: string
|
||||
listed: number
|
||||
known: number
|
||||
added: number
|
||||
password: boolean
|
||||
}
|
||||
|
||||
/** Filename of an NZB entry, from its subject (tolerating odd quoting). */
|
||||
export function nzbFileName(subject: string): string {
|
||||
const parsed = parseSubject(subject)
|
||||
const name = parsed?.name ?? /"([^"]+)"/.exec(subject)?.[1] ?? subject
|
||||
return name.trim().replace(/^"+|"+$/g, '').trim() || subject.trim()
|
||||
}
|
||||
|
||||
export interface NzbImportOptions {
|
||||
/**
|
||||
* For NZBs inside an archive: the entry's name. It keeps several NZBs of
|
||||
* one archive apart and names their release when the NZB has no name.
|
||||
*/
|
||||
entryName?: string
|
||||
/** Move the posted file into the release the NZB describes (default true). */
|
||||
moveContainer?: boolean
|
||||
/** A name given with the NZB (e.g. a Spotnet title); wins over the NZB's own. */
|
||||
name?: string
|
||||
/** Release source for new releases (default `nzb`). */
|
||||
source?: ReleaseIdentity['source']
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads an NZB that was posted to Usenet into the index (CONCEPT.md D26).
|
||||
* Files it lists that are already known (by first segment) keep their
|
||||
* release, which the NZB then joins; the others are added as files, so posts
|
||||
* spread over groups we do not index still become complete.
|
||||
*/
|
||||
class NzbImportService {
|
||||
async import(nzbFile: any, xml: string, options: NzbImportOptions = {}): Promise<NzbImportResult | null> {
|
||||
const nzb = parseNzb(xml)
|
||||
if (!nzb || nzb.files.length === 0) return null
|
||||
|
||||
const entries = nzb.files.map((f) => ({
|
||||
file: f,
|
||||
name: nzbFileName(f.subject),
|
||||
firstId: f.segments[0].id,
|
||||
}))
|
||||
|
||||
return db.transaction(async (trx) => {
|
||||
const known = await trx
|
||||
.from('files')
|
||||
.whereRaw(`(message_ids->'1'->>'id') = ANY(?)`, [entries.map((e) => e.firstId)])
|
||||
.select('id', 'release_id', `message_ids`)
|
||||
const knownIds = new Set(known.map((f) => f.message_ids?.['1']?.id))
|
||||
|
||||
// Join the release the listed files are already in, else the posted NZB's own, else a new one.
|
||||
const counts = new Map<number, number>()
|
||||
for (const f of known) if (f.release_id) counts.set(Number(f.release_id), (counts.get(Number(f.release_id)) ?? 0) + 1)
|
||||
let releaseId: number | undefined = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
|
||||
const title = options.name?.trim() || this.title(nzb, nzbFile, entries.map((e) => e.name), options.entryName)
|
||||
if (!releaseId) {
|
||||
const origin = `nzb\n${nzbFile.message_ids?.['1']?.id ?? nzbFile.id}`
|
||||
const key = createHash('sha1').update(options.entryName ? `${origin}\n${options.entryName}` : origin).digest('hex')
|
||||
const ids = await IngestService.upsertReleases(trx.knexClient, [
|
||||
{
|
||||
release: { key, name: title, source: options.source ?? 'nzb' },
|
||||
poster: nzbFile.poster,
|
||||
groups: nzbFile.groups ?? [],
|
||||
postedAt: nzbFile.date ? new Date(Number(nzbFile.date)) : undefined,
|
||||
},
|
||||
])
|
||||
releaseId = ids.get(key)!
|
||||
}
|
||||
|
||||
const now = new Date()
|
||||
const missing = entries.filter((e) => !knownIds.has(e.firstId))
|
||||
for (let i = 0; i < missing.length; i += 500) {
|
||||
await trx
|
||||
.knexClient('files')
|
||||
.insert(
|
||||
missing.slice(i, i + 500).map(({ file, name, firstId }) => ({
|
||||
file_key: createHash('sha1').update(`nzb-file\n${firstId}`).digest('hex'),
|
||||
filename: name,
|
||||
poster: file.poster || nzbFile.poster,
|
||||
date: (file.date || Math.floor(Number(nzbFile.date) / 1000)) * 1000,
|
||||
parts: file.segments.length,
|
||||
size: file.segments.reduce((sum, s) => sum + s.bytes, 0),
|
||||
message_ids: JSON.stringify(Object.fromEntries(file.segments.map((s) => [s.number, { id: s.id, size: s.bytes }]))),
|
||||
groups: JSON.stringify(file.groups),
|
||||
release_id: releaseId,
|
||||
created_at: now,
|
||||
updated_at: now,
|
||||
}))
|
||||
)
|
||||
.onConflict('file_key')
|
||||
.ignore()
|
||||
}
|
||||
// The poster's NZB says which files form the post: known files join this
|
||||
// release, also when name rules had split them over several.
|
||||
const previous = [...new Set(known.map((f) => f.release_id).filter((id) => id && Number(id) !== releaseId))]
|
||||
await trx
|
||||
.from('files')
|
||||
.whereIn('id', known.filter((f) => Number(f.release_id) !== releaseId).map((f) => f.id))
|
||||
.update({ release_id: releaseId })
|
||||
if (previous.length) {
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET release_id = ? FROM files f WHERE b.file_id = f.id AND f.release_id = ? AND b.release_id IS DISTINCT FROM ?`,
|
||||
[releaseId, releaseId, releaseId]
|
||||
)
|
||||
await trx.from('releases').whereIn('id', previous).where('file_count', 0).delete()
|
||||
}
|
||||
// The NZB file joins the release it describes, unless it is part of a real release already.
|
||||
const own = nzbFile.release_id ? Number(nzbFile.release_id) : null
|
||||
if (own !== releaseId && options.moveContainer !== false) {
|
||||
const ownRelease = own ? await trx.from('releases').where('id', own).first() : null
|
||||
if (!ownRelease || ownRelease.file_count <= 1) {
|
||||
await trx.from('files').where('id', nzbFile.id).update({ release_id: releaseId })
|
||||
if (own) await trx.from('releases').where('id', own).where('file_count', 0).delete()
|
||||
}
|
||||
}
|
||||
|
||||
const release = await trx.from('releases').where('id', releaseId).first()
|
||||
const update: Record<string, unknown> = {}
|
||||
if (nzb.meta.password) update.password = nzb.meta.password
|
||||
if (!release.file_total) update.file_total = entries.length
|
||||
if ((looksRandom(release.name) || release.source === 'post') && !looksRandom(title)) update.name = title
|
||||
if (Object.keys(update).length) await trx.from('releases').where('id', releaseId).update(update)
|
||||
|
||||
const result = {
|
||||
releaseId,
|
||||
releaseName: (update.name as string) ?? release.name,
|
||||
listed: entries.length,
|
||||
known: known.length,
|
||||
added: missing.length,
|
||||
password: Boolean(nzb.meta.password),
|
||||
}
|
||||
logger.info(
|
||||
`NZB "${options.entryName ? `${nzbFile.filename}:${options.entryName}` : nzbFile.filename}" → release ${releaseId} "${result.releaseName}": ${result.listed} files listed, ` +
|
||||
`${result.known} known, ${result.added} added${result.password ? ', with password' : ''}.`
|
||||
)
|
||||
return result
|
||||
})
|
||||
}
|
||||
|
||||
/** Release name: the NZB's meta name, else the NZB's (entry or file) name, else its largest file's stem. */
|
||||
private title(nzb: ParsedNzb, nzbFile: any, names: string[], entryName?: string): string {
|
||||
if (nzb.meta.name) return nzb.meta.name
|
||||
const entry = entryName ? fileStem(entryName.split(/[\\/]/).pop() ?? entryName) : ''
|
||||
if (entry && !looksRandom(entry)) return entry
|
||||
const own = fileStem(String(nzbFile.real_name ?? nzbFile.filename ?? ''))
|
||||
if (own && !looksRandom(own)) return own
|
||||
const largest = nzb.files
|
||||
.map((f, i) => ({ name: names[i], bytes: f.segments.reduce((s, x) => s + x.bytes, 0) }))
|
||||
.sort((a, b) => b.bytes - a.bytes)[0]
|
||||
return fileStem(largest?.name ?? own)
|
||||
}
|
||||
}
|
||||
|
||||
export default new NzbImportService()
|
||||
@@ -0,0 +1,165 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NntpService from '#services/NntpService'
|
||||
import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
|
||||
import { decodeSegment, YencFile } from '#lib/yenc/index'
|
||||
import { isPar2, parsePar2, type Par2Set } from '#lib/par2/index'
|
||||
import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
|
||||
import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'
|
||||
|
||||
/** Par2 index files are small; recovery volumes can be large and are not needed. */
|
||||
const MAX_PARTS = 20
|
||||
/** Size-only matching below this is too ambiguous. */
|
||||
const SIZE_ONLY_MIN = 1024 * 1024
|
||||
|
||||
function countBy<T>(values: T[]): Map<T, number> {
|
||||
const counts = new Map<T, number>()
|
||||
for (const value of values) counts.set(value, (counts.get(value) ?? 0) + 1)
|
||||
return counts
|
||||
}
|
||||
|
||||
export interface Par2Result {
|
||||
releaseId: number
|
||||
releaseName: string
|
||||
listed: number
|
||||
matched: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Recognises releases of obfuscated posts from their par2 files: the par2
|
||||
* lists the real name and exact size of every file in the set, and
|
||||
* body-derived binaries carry their exact size. See CONCEPT.md (D16).
|
||||
*/
|
||||
class Par2Service {
|
||||
/**
|
||||
* Reads a par2 file whose first segment is already decoded, downloading the
|
||||
* remaining parts if there are any. Returns null when it is not par2.
|
||||
*/
|
||||
async read(first: Buffer, messageIds: Record<string, { id: string }>): Promise<Par2Set | null> {
|
||||
if (!isPar2(first)) return null
|
||||
const ids = Object.entries(messageIds)
|
||||
.sort(([a], [b]) => Number(a) - Number(b))
|
||||
.map(([, part]) => part.id)
|
||||
if (ids.length > MAX_PARTS) return null
|
||||
if (ids.length === 1) return parsePar2(first)
|
||||
|
||||
let assembled: YencFile | undefined
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
for (const id of ids) {
|
||||
const segment = decodeSegment(await NntpService.fetchBody(conn, id))
|
||||
assembled ??= YencFile.fromSegment(segment)
|
||||
assembled.add(segment)
|
||||
}
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
return parsePar2(assembled!.getBuffer())
|
||||
}
|
||||
|
||||
async attach(binary: any, set: Par2Set): Promise<Par2Result> {
|
||||
const result = await db.transaction(async (trx) => {
|
||||
// Exact: size and first-16-KiB MD5 of a listed file. Only these may pull
|
||||
// the set into a release the files already belong to; size alone is
|
||||
// ambiguous across posts (standard volume sizes recur).
|
||||
const exact = await trx
|
||||
.from('binaries')
|
||||
.whereIn(['size', 'md5_16k'] as any, set.files.map((f) => [f.length, f.md5_16k]) as any)
|
||||
.select('id', 'release_id')
|
||||
|
||||
// Size only, for files without a fingerprint: the size must be listed
|
||||
// once in this set, be large, and belong to exactly one binary nearby.
|
||||
const listedOnce = [...countBy(set.files.map((f) => f.length))]
|
||||
.filter(([size, n]) => n === 1 && size >= SIZE_ONLY_MIN)
|
||||
.map(([size]) => size)
|
||||
const bySize = listedOnce.length
|
||||
? await trx
|
||||
.from('binaries')
|
||||
.whereIn('size', listedOnce)
|
||||
.whereNull('md5_16k')
|
||||
.whereRaw('abs(extract(epoch FROM (posted_at - ?::timestamptz))) < ?', [
|
||||
binary.posted_at,
|
||||
PAR2_MATCH_WINDOW_SECONDS,
|
||||
])
|
||||
.select('id', 'release_id', 'size')
|
||||
: []
|
||||
const sizeCounts = countBy(bySize.map((b) => Number(b.size)))
|
||||
const fuzzy = bySize.filter((b) => sizeCounts.get(Number(b.size)) === 1)
|
||||
const candidates = [...exact, ...fuzzy]
|
||||
|
||||
// Prefer a release the exactly matched files are already in (e.g.
|
||||
// obfuscated volumes grouped by their stem); otherwise the set's own.
|
||||
const counts = countBy(exact.filter((c) => c.release_id).map((c) => Number(c.release_id)))
|
||||
let releaseId = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0]
|
||||
const identity = par2ReleaseIdentity(
|
||||
set.recoverySetId,
|
||||
set.files.map((f) => f.name)
|
||||
)
|
||||
if (!releaseId) {
|
||||
const ids = await IngestService.upsertReleases(trx.knexClient, [
|
||||
{
|
||||
release: identity,
|
||||
poster: binary.poster,
|
||||
weakIdentity: true,
|
||||
groups: binary.groups,
|
||||
postedAt: binary.posted_at ? new Date(binary.posted_at) : undefined,
|
||||
},
|
||||
])
|
||||
releaseId = ids.get(identity.key)!
|
||||
}
|
||||
|
||||
await trx
|
||||
.knexClient('release_files')
|
||||
.insert(
|
||||
set.files.map((f) => ({
|
||||
release_id: releaseId,
|
||||
par2_file_id: f.fileId,
|
||||
name: f.name,
|
||||
size: f.length,
|
||||
md5_16k: f.md5_16k,
|
||||
md5: f.md5,
|
||||
set_id: set.recoverySetId,
|
||||
}))
|
||||
)
|
||||
.onConflict(['release_id', 'par2_file_id'])
|
||||
.ignore()
|
||||
|
||||
const matchIds = candidates.filter((c) => !c.release_id).map((c) => Number(c.id))
|
||||
await trx.from('binaries').whereIn('id', [binary.id, ...matchIds]).whereNull('release_id').update({ release_id: releaseId })
|
||||
// Exact matches sitting alone in a random-named release (one file of a
|
||||
// season, each posted under its own random name) join the set's release.
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries b SET release_id = ? WHERE b.id = ANY(?) AND b.release_id <> ? AND ${WEAK_RELEASE_SQL('b.release_id')}`,
|
||||
[releaseId, exact.map((c) => Number(c.id)), releaseId]
|
||||
)
|
||||
await trx.rawQuery(
|
||||
`UPDATE files f SET release_id = b.release_id FROM binaries b
|
||||
WHERE b.file_id = f.id AND b.release_id = ? AND f.release_id IS DISTINCT FROM b.release_id`,
|
||||
[releaseId]
|
||||
)
|
||||
|
||||
// The set may have joined a release named after one random filename.
|
||||
let releaseName = (await trx.from('releases').where('id', releaseId).first())?.name
|
||||
const better = par2ReleaseName(set.files.map((f) => f.name))
|
||||
if (releaseName && better && isBetterName(releaseName, better)) {
|
||||
await trx.from('releases').where('id', releaseId).update({ name: better })
|
||||
releaseName = better
|
||||
}
|
||||
logger.info(
|
||||
`Par2 set ${set.recoverySetId.slice(0, 8)} → release ${releaseId} "${releaseName}": ` +
|
||||
`${set.files.length} files listed, ${matchIds.length} binaries matched by size.`
|
||||
)
|
||||
return { releaseId, releaseName, listed: set.files.length, matched: matchIds.length }
|
||||
})
|
||||
|
||||
// Real names for every known file of the set (exact fingerprint, or size within the release).
|
||||
await NameService.resolveForRelease(result.releaseId)
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
export default new Par2Service()
|
||||
@@ -1,29 +1,60 @@
|
||||
import { Queue } from 'bullmq'
|
||||
import { Queue, type DefaultJobOptions } from 'bullmq'
|
||||
import queueConfig from '#config/queue'
|
||||
import gcConfig from '#config/gc'
|
||||
|
||||
const defaultJobOptions: DefaultJobOptions = {
|
||||
// Transient NNTP errors (e.g. a pooled connection the server closed) are retried.
|
||||
attempts: 3,
|
||||
backoff: { type: 'exponential', delay: 2000 },
|
||||
removeOnComplete: {
|
||||
age: gcConfig.jobs.completedAgeSeconds,
|
||||
count: gcConfig.jobs.completedMaxCount,
|
||||
},
|
||||
removeOnFail: { age: gcConfig.jobs.failedAgeSeconds },
|
||||
}
|
||||
|
||||
/**
|
||||
* Queues are created (and connect to Redis) on first use. Ace imports every
|
||||
* command at boot, so eager queues would keep one-off commands from exiting.
|
||||
*/
|
||||
class QueueService {
|
||||
public readonly nntpFetchQueue: Queue
|
||||
public readonly headerQueue: Queue
|
||||
public readonly fileQueue: Queue
|
||||
public readonly bodyQueue: Queue
|
||||
public readonly collectionQueue: Queue
|
||||
private readonly queues = new Map<string, Queue>()
|
||||
|
||||
constructor() {
|
||||
this.nntpFetchQueue = new Queue('nntp-fetch-queue', { connection: queueConfig.connection })
|
||||
this.headerQueue = new Queue('header-queue', { connection: queueConfig.connection })
|
||||
this.fileQueue = new Queue('file-queue', { connection: queueConfig.connection })
|
||||
this.bodyQueue = new Queue('body-queue', { connection: queueConfig.connection })
|
||||
this.collectionQueue = new Queue('collection-queue', { connection: queueConfig.connection })
|
||||
private queue(name: string): Queue {
|
||||
let queue = this.queues.get(name)
|
||||
if (!queue) {
|
||||
queue = new Queue(name, { connection: queueConfig.connection, defaultJobOptions })
|
||||
this.queues.set(name, queue)
|
||||
}
|
||||
return queue
|
||||
}
|
||||
|
||||
get nntpFetchQueue() {
|
||||
return this.queue('nntp-fetch-queue')
|
||||
}
|
||||
get fileQueue() {
|
||||
return this.queue('file-queue')
|
||||
}
|
||||
get bodyQueue() {
|
||||
return this.queue('body-queue')
|
||||
}
|
||||
get collectionQueue() {
|
||||
return this.queue('collection-queue')
|
||||
}
|
||||
|
||||
/** Queues no longer used by the pipeline, kept here so GC can empty them. */
|
||||
get legacy(): Queue[] {
|
||||
return [this.queue('header-queue')]
|
||||
}
|
||||
|
||||
get all(): Queue[] {
|
||||
return [this.nntpFetchQueue, this.fileQueue, this.bodyQueue, this.collectionQueue]
|
||||
}
|
||||
|
||||
/** Closes the queues that were opened. */
|
||||
async closeAll() {
|
||||
await Promise.all([
|
||||
this.nntpFetchQueue.close(),
|
||||
this.headerQueue.close(),
|
||||
this.fileQueue.close(),
|
||||
this.bodyQueue.close(),
|
||||
this.collectionQueue.close(),
|
||||
])
|
||||
await Promise.all([...this.queues.values()].map((queue) => queue.close()))
|
||||
this.queues.clear()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,11 +4,21 @@ import redisConfig from '#config/redis'
|
||||
const require = createRequire(import.meta.url)
|
||||
const IORedis = require('ioredis')
|
||||
|
||||
/**
|
||||
* Connects on first use. Ace imports every command at boot, so an eager
|
||||
* connection would keep even `node ace list` from exiting.
|
||||
*/
|
||||
class RedisService {
|
||||
public readonly client: any
|
||||
private instance?: any
|
||||
|
||||
constructor() {
|
||||
this.client = new IORedis(redisConfig.connections.main)
|
||||
public get client(): any {
|
||||
this.instance ??= new IORedis(redisConfig.connections.main)
|
||||
return this.instance
|
||||
}
|
||||
|
||||
async close() {
|
||||
if (this.instance) await this.instance.quit()
|
||||
this.instance = undefined
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,284 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import { buildNzb } from '#lib/nzb/index'
|
||||
import SpotService from '#services/SpotService'
|
||||
|
||||
export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
|
||||
|
||||
export interface SearchParams {
|
||||
q?: string
|
||||
/** Only releases whose name is readable (not an obfuscated token). */
|
||||
readable?: boolean
|
||||
/** Only releases that have all their files. */
|
||||
complete?: boolean
|
||||
/** Hide password-protected releases. */
|
||||
hideEncrypted?: boolean
|
||||
source?: string
|
||||
poster?: string
|
||||
sort?: SortOrder
|
||||
page?: number
|
||||
perPage?: number
|
||||
/** Newznab: exact category ids (already expanded from parents). */
|
||||
categories?: number[]
|
||||
/** Newznab: leave out releases known to be missing files. */
|
||||
notIncomplete?: boolean
|
||||
maxAgeDays?: number
|
||||
/** TV: season and episode numbers, matched as S01E02 or 1x02 in the name. */
|
||||
season?: number
|
||||
episode?: number
|
||||
/** Overrides `page` (Newznab uses offsets). */
|
||||
offset?: number
|
||||
/** Leave out releases that are only an encrypted container (nzbe). */
|
||||
hideContainers?: boolean
|
||||
/** Include incomplete releases that consist of par2 files only (hidden by default). */
|
||||
includePar2Only?: boolean
|
||||
}
|
||||
|
||||
const SORTS: Record<SortOrder, string> = {
|
||||
newest: 'r.posted_at DESC NULLS LAST, r.id DESC',
|
||||
oldest: 'r.posted_at ASC NULLS LAST, r.id ASC',
|
||||
largest: 'r.size DESC, r.id DESC',
|
||||
files: 'r.file_count DESC, r.id DESC',
|
||||
name: 'r.name ASC, r.id ASC',
|
||||
}
|
||||
|
||||
/**
|
||||
* Shown in search: releases with files, and Spotnet listings whose NZB is
|
||||
* fetched on first download (CONCEPT.md D29). Matches the partial indexes.
|
||||
*/
|
||||
const VISIBLE_SQL = `(r.file_count > 0 OR r.source = 'spot')`
|
||||
|
||||
/** Incomplete, and every file is a par2 file: nothing to download but repair data. */
|
||||
const PAR2_ONLY_SQL = `(r.file_count > 0 AND NOT r.complete AND NOT EXISTS (
|
||||
SELECT 1 FROM files f WHERE f.release_id = r.id
|
||||
AND f.content_type IS DISTINCT FROM 'par2' AND f.filename !~* '\\.par2$'))`
|
||||
|
||||
/** A search word as an ILIKE pattern; dots, underscores, dashes and spaces are interchangeable. */
|
||||
function pattern(word: string): string {
|
||||
const escaped = word.replace(/[\\%]/g, (c) => `\\${c}`)
|
||||
return `%${escaped.replace(/[._\- ]+/g, '_')}%`
|
||||
}
|
||||
|
||||
/**
|
||||
* Searching and browsing releases for the web interface (CONCEPT.md D22).
|
||||
*/
|
||||
class SearchService {
|
||||
async search(params: SearchParams) {
|
||||
const where: string[] = [
|
||||
VISIBLE_SQL,
|
||||
// Retracted by its poster or a Spotnet moderator (CONCEPT.md D29).
|
||||
`NOT EXISTS (SELECT 1 FROM spots s WHERE s.release_id = r.id AND s.status = 'removed')`,
|
||||
]
|
||||
const bindings: unknown[] = []
|
||||
|
||||
const words = (params.q ?? '').trim().split(/\s+/).filter(Boolean).slice(0, 8)
|
||||
for (const word of words) {
|
||||
const p = pattern(word)
|
||||
// Every word must match the release name, a file name, or the archive contents.
|
||||
// One trigram-indexed lookup per table, not a subquery per release.
|
||||
where.push(`(r.name ILIKE ? OR r.contents::text ILIKE ?
|
||||
OR r.id IN (SELECT f.release_id FROM files f WHERE f.filename ILIKE ? OR f.real_name ILIKE ?))`)
|
||||
bindings.push(p, p, p, p)
|
||||
}
|
||||
if (params.readable) where.push('is_readable_name(r.name)')
|
||||
if (params.hideEncrypted) where.push('r.encrypted IS NOT TRUE')
|
||||
if (params.hideContainers) where.push('r.container IS NULL')
|
||||
// par2 files whose data never arrived (e.g. uploads spread over groups we can't fetch).
|
||||
if (!params.includePar2Only) where.push(`(${PAR2_ONLY_SQL}) IS NOT TRUE`)
|
||||
// Kept current by triggers (migration …860).
|
||||
if (params.complete) where.push('r.complete')
|
||||
if (params.source) {
|
||||
where.push('r.source = ?')
|
||||
bindings.push(params.source)
|
||||
}
|
||||
if (params.poster) {
|
||||
where.push('r.poster ILIKE ?')
|
||||
bindings.push(pattern(params.poster))
|
||||
}
|
||||
if (params.categories?.length) {
|
||||
where.push('r.category = ANY(?)')
|
||||
bindings.push(params.categories)
|
||||
}
|
||||
if (params.notIncomplete) {
|
||||
where.push(`(r.complete OR (r.file_total IS NULL AND NOT EXISTS (SELECT 1 FROM release_files rf WHERE rf.release_id = r.id)))`)
|
||||
}
|
||||
if (params.maxAgeDays) {
|
||||
where.push(`r.posted_at >= now() - make_interval(days => ?)`)
|
||||
bindings.push(params.maxAgeDays)
|
||||
}
|
||||
if (params.season !== undefined) {
|
||||
const s = params.season
|
||||
const e = params.episode
|
||||
where.push('r.name ~* ?')
|
||||
bindings.push(
|
||||
e !== undefined
|
||||
? `(^|[._ -])(S0*${s}E0*${e}|0*${s}x0*${e})([^0-9]|$)`
|
||||
: `(^|[._ -])(S0*${s}([._ -]|E\\d|$)|0*${s}x\\d)`
|
||||
)
|
||||
}
|
||||
|
||||
const perPage = Math.min(Math.max(params.perPage ?? 50, 1), 200)
|
||||
const page = Math.max(params.page ?? 1, 1)
|
||||
const offset = params.offset ?? (page - 1) * perPage
|
||||
const order = SORTS[params.sort ?? 'newest'] ?? SORTS.newest
|
||||
const whereSql = where.join(' AND ')
|
||||
|
||||
const [{ rows }, { rows: countRows }] = await Promise.all([
|
||||
db.rawQuery(
|
||||
`SELECT r.id, r.name, r.source, r.poster, r.groups, r.posted_at, r.file_total, r.file_count, r.category, r.container,
|
||||
r.size, r.encrypted, is_readable_name(r.name) AS readable,
|
||||
(SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id) AS listed_files,
|
||||
(SELECT count(*) FROM release_files rf WHERE rf.release_id = r.id AND EXISTS (
|
||||
SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = r.id AND (b.size = rf.size OR f.real_name = rf.name))) AS listed_present,
|
||||
jsonb_array_length(coalesce(r.contents, '[]'::jsonb)) AS archive_entries,
|
||||
r.complete, ${PAR2_ONLY_SQL} AS par2_only
|
||||
FROM releases r WHERE ${whereSql}
|
||||
ORDER BY ${order} LIMIT ? OFFSET ?`,
|
||||
[...bindings, perPage, offset]
|
||||
),
|
||||
db.rawQuery(`SELECT count(*) AS total FROM releases r WHERE ${whereSql}`, bindings),
|
||||
])
|
||||
|
||||
return {
|
||||
page,
|
||||
perPage,
|
||||
offset,
|
||||
total: Number(countRows[0].total),
|
||||
releases: rows.map((r: any) => ({
|
||||
id: Number(r.id),
|
||||
name: r.name,
|
||||
source: r.source,
|
||||
poster: r.poster,
|
||||
groups: r.groups,
|
||||
postedAt: r.posted_at,
|
||||
fileTotal: r.file_total,
|
||||
fileCount: r.file_count,
|
||||
size: Number(r.size),
|
||||
encrypted: r.encrypted,
|
||||
readable: r.readable,
|
||||
listedFiles: Number(r.listed_files),
|
||||
listedPresent: Number(r.listed_present),
|
||||
archiveEntries: Number(r.archive_entries),
|
||||
complete: r.complete,
|
||||
category: r.category,
|
||||
container: r.container,
|
||||
par2Only: r.par2_only,
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
async release(id: number) {
|
||||
const release = await db.from('releases').where('id', id).first()
|
||||
if (!release) return null
|
||||
|
||||
const [files, listed, pending] = await Promise.all([
|
||||
db
|
||||
.from('files as f')
|
||||
.leftJoin('binaries as b', 'b.file_id', 'f.id')
|
||||
.where('f.release_id', id)
|
||||
.orderByRaw('coalesce(f.real_name, f.filename)')
|
||||
.select('f.id', 'f.filename', 'f.real_name', 'f.size', 'f.parts', 'f.date', 'f.poster', 'b.size as exact_size'),
|
||||
db.rawQuery(
|
||||
`SELECT rf.name, rf.size,
|
||||
EXISTS (SELECT 1 FROM files f JOIN binaries b ON b.file_id = f.id
|
||||
WHERE f.release_id = rf.release_id AND (f.real_name = rf.name OR b.size = rf.size)) AS present
|
||||
FROM release_files rf WHERE rf.release_id = ? ORDER BY rf.name`,
|
||||
[id]
|
||||
),
|
||||
db.rawQuery(
|
||||
`SELECT count(*) AS binaries, coalesce(sum(b.total_parts), 0) AS parts_total,
|
||||
coalesce(sum((SELECT count(*) FROM parts p WHERE p.binary_id = b.id)), 0) AS parts_have
|
||||
FROM binaries b WHERE b.release_id = ? AND b.completed_at IS NULL AND b.conflicted_at IS NULL`,
|
||||
[id]
|
||||
),
|
||||
])
|
||||
|
||||
const spot = release.source === 'spot'
|
||||
? await db.from('spots').where('release_id', id).select('tag', 'poster', 'category', 'website').first()
|
||||
: null
|
||||
return {
|
||||
id: Number(release.id),
|
||||
name: release.name,
|
||||
source: release.source,
|
||||
spot: spot ? { tag: spot.tag, poster: spot.poster, category: spot.category, website: spot.website } : null,
|
||||
listingOnly: release.source === 'spot' && Number(release.file_count) === 0,
|
||||
poster: release.poster,
|
||||
groups: release.groups,
|
||||
postedAt: release.posted_at,
|
||||
fileTotal: release.file_total,
|
||||
fileCount: release.file_count,
|
||||
size: Number(release.size),
|
||||
encrypted: release.encrypted,
|
||||
container: release.container,
|
||||
contents: release.contents ?? [],
|
||||
files: files.map((f) => ({
|
||||
id: Number(f.id),
|
||||
name: f.real_name ?? f.filename,
|
||||
postedName: f.filename,
|
||||
realName: f.real_name,
|
||||
size: Number(f.exact_size ?? f.size),
|
||||
parts: f.parts,
|
||||
postedAt: f.date ? new Date(Number(f.date)) : null,
|
||||
poster: f.poster,
|
||||
})),
|
||||
par2Files: listed.rows.map((r: any) => ({ name: r.name, size: Number(r.size), present: r.present })),
|
||||
pending: {
|
||||
binaries: Number(pending.rows[0].binaries),
|
||||
partsHave: Number(pending.rows[0].parts_have),
|
||||
partsTotal: Number(pending.rows[0].parts_total),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
async nzb(id: number): Promise<{ name: string; xml: string } | null> {
|
||||
let release = await db.from('releases').where('id', id).first()
|
||||
if (!release) return null
|
||||
if (release.source === 'spot' && Number(release.file_count) === 0) {
|
||||
// A Spotnet listing: read its NZB from Usenet now (the import may join known files elsewhere).
|
||||
const imported = await SpotService.importListing(id)
|
||||
if (!imported) return null
|
||||
id = imported
|
||||
release = await db.from('releases').where('id', id).first()
|
||||
}
|
||||
const files = await db
|
||||
.from('files')
|
||||
.where('release_id', id)
|
||||
.orderByRaw('coalesce(real_name, filename)')
|
||||
.select('filename', 'real_name', 'poster', 'date', 'groups', 'message_ids')
|
||||
const xml = buildNzb(
|
||||
files.map((f) => ({
|
||||
name: f.real_name ?? f.filename,
|
||||
poster: f.poster,
|
||||
date: Number(f.date),
|
||||
groups: f.groups?.length ? f.groups : ['alt.binaries.test'],
|
||||
segments: f.message_ids,
|
||||
})),
|
||||
{ title: release.name, password: release.password ?? undefined }
|
||||
)
|
||||
return { name: release.name, xml }
|
||||
}
|
||||
|
||||
async stats() {
|
||||
const { rows } = await db.rawQuery(`
|
||||
SELECT (SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL}) AS releases,
|
||||
(SELECT count(*) FROM releases r WHERE ${VISIBLE_SQL} AND is_readable_name(name)) AS readable_releases,
|
||||
(SELECT count(*) FROM files) AS files,
|
||||
(SELECT coalesce(sum(size), 0) FROM files) AS size,
|
||||
(SELECT count(*) FROM releases WHERE encrypted) AS encrypted_releases,
|
||||
(SELECT max(updated_at) FROM groups) AS last_indexed_at,
|
||||
(SELECT json_agg(json_build_object('source', source, 'releases', n) ORDER BY n DESC)
|
||||
FROM (SELECT coalesce(source, 'unknown') AS source, count(*) AS n FROM releases r WHERE ${VISIBLE_SQL} GROUP BY 1) s) AS sources`)
|
||||
const r = rows[0]
|
||||
return {
|
||||
releases: Number(r.releases),
|
||||
readableReleases: Number(r.readable_releases),
|
||||
files: Number(r.files),
|
||||
size: Number(r.size),
|
||||
encryptedReleases: Number(r.encrypted_releases),
|
||||
lastIndexedAt: r.last_indexed_at,
|
||||
sources: r.sources ?? [],
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new SearchService()
|
||||
@@ -0,0 +1,45 @@
|
||||
import { setTimeout as sleep } from 'node:timers/promises'
|
||||
|
||||
/** After a signal, give cleanup this long before exiting anyway. */
|
||||
const FORCE_EXIT_MS = 25_000
|
||||
|
||||
/**
|
||||
* Graceful shutdown for the long-running commands. SIGTERM (docker stop) and
|
||||
* SIGINT wake up `wait()` and `sleep()`, so workers can finish their active
|
||||
* jobs and close their connections before the process exits.
|
||||
*/
|
||||
class Shutdown {
|
||||
private readonly controller = new AbortController()
|
||||
|
||||
constructor() {
|
||||
for (const signal of ['SIGTERM', 'SIGINT'] as const) {
|
||||
process.once(signal, () => {
|
||||
this.controller.abort()
|
||||
setTimeout(() => process.exit(0), FORCE_EXIT_MS).unref()
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
get requested(): boolean {
|
||||
return this.controller.signal.aborted
|
||||
}
|
||||
|
||||
/** Resolves when a shutdown signal arrives. */
|
||||
wait(): Promise<void> {
|
||||
if (this.requested) return Promise.resolve()
|
||||
return new Promise((resolve) =>
|
||||
this.controller.signal.addEventListener('abort', () => resolve(), { once: true })
|
||||
)
|
||||
}
|
||||
|
||||
/** Sleeps, but returns early on shutdown. */
|
||||
async sleep(ms: number): Promise<void> {
|
||||
try {
|
||||
await sleep(ms, undefined, { signal: this.controller.signal })
|
||||
} catch {
|
||||
// aborted by a shutdown signal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new Shutdown()
|
||||
@@ -0,0 +1,146 @@
|
||||
import { createHash } from 'node:crypto'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import IngestService from '#services/IngestService'
|
||||
import NameService from '#services/NameService'
|
||||
import { splitVerified } from '#lib/split/index'
|
||||
|
||||
/** Conflicted binaries handled per run; the rest follow on the next tick. */
|
||||
const BATCH = 200
|
||||
|
||||
export interface SplitResult {
|
||||
parents: number
|
||||
volumes: number
|
||||
completed: number
|
||||
}
|
||||
|
||||
/**
|
||||
* Splits conflicted obfuscated binaries (several equal-sized files under one
|
||||
* size+parts identity) into one binary per volume. Only volumes whose
|
||||
* combined part CRCs match the file CRC of their last part are created, so a
|
||||
* split can never produce a corrupt file; the rest waits for more articles.
|
||||
* See CONCEPT.md (D20).
|
||||
*/
|
||||
class SplitService {
|
||||
async splitPending(): Promise<SplitResult> {
|
||||
const parents = await db
|
||||
.from('binaries')
|
||||
.whereNotNull('conflicted_at')
|
||||
.whereNull('parent_binary_id')
|
||||
// split_at = 'infinity' marks binaries that cannot be split (see migration …810).
|
||||
.where((q) => q.whereNull('split_at').orWhereRaw('last_seen_at > split_at'))
|
||||
// Most recently active first: those are the posts still coming in.
|
||||
.orderBy('last_seen_at', 'desc')
|
||||
.limit(BATCH)
|
||||
|
||||
const volumeIds: number[] = []
|
||||
for (const parent of parents) {
|
||||
volumeIds.push(...(await this.split(parent)))
|
||||
}
|
||||
|
||||
// Former conflicts that turned out to be reposts (unflagged, keep their
|
||||
// split_at): no new article may arrive to trigger their completion.
|
||||
const unflagged = await db
|
||||
.from('binaries')
|
||||
.whereNull('conflicted_at')
|
||||
.whereNull('completed_at')
|
||||
.whereNull('parent_binary_id')
|
||||
.whereNotNull('split_at')
|
||||
.whereRaw("split_at <> 'infinity'")
|
||||
.limit(BATCH)
|
||||
.select('id')
|
||||
volumeIds.push(...unflagged.map((b) => Number(b.id)))
|
||||
|
||||
await IngestService.matchPar2Releases(volumeIds)
|
||||
await NameService.resolve(volumeIds)
|
||||
const completed = await IngestService.claimCompleted(volumeIds)
|
||||
await IngestService.enqueue(completed)
|
||||
return { parents: parents.length, volumes: volumeIds.length, completed: completed.length }
|
||||
}
|
||||
|
||||
/** Creates the newly verified volumes of one conflicted binary; returns their ids. */
|
||||
private async split(parent: any): Promise<number[]> {
|
||||
return db.transaction(async (trx) => {
|
||||
// Unverified volumes from before verification existed can never complete.
|
||||
await trx.from('binaries').where('parent_binary_id', parent.id).whereNull('completed_at').delete()
|
||||
|
||||
const { rows } = await trx.rawQuery(
|
||||
`WITH taken AS (
|
||||
SELECT p.message_id FROM parts p JOIN binaries v ON v.id = p.binary_id
|
||||
WHERE v.parent_binary_id = ?
|
||||
), candidates AS (
|
||||
SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM parts WHERE binary_id = ?
|
||||
UNION ALL
|
||||
SELECT number, message_id, bytes, article_number, pcrc32, part_size, file_crc32 FROM part_conflicts WHERE binary_id = ?
|
||||
)
|
||||
SELECT * FROM candidates WHERE message_id NOT IN (SELECT message_id FROM taken)`,
|
||||
[parent.id, parent.id, parent.id]
|
||||
)
|
||||
const volumes = splitVerified(
|
||||
rows.map((r: any) => ({
|
||||
number: r.number,
|
||||
messageId: r.message_id,
|
||||
bytes: r.bytes,
|
||||
articleNumber: r.article_number === null ? null : Number(r.article_number),
|
||||
pcrc32: r.pcrc32,
|
||||
partSize: r.part_size,
|
||||
fileCrc32: r.file_crc32,
|
||||
})),
|
||||
{ totalParts: parent.total_parts }
|
||||
)
|
||||
|
||||
const now = new Date()
|
||||
const ids: number[] = []
|
||||
for (const articles of volumes) {
|
||||
// A verified volume is fixed, so its first article identifies it.
|
||||
const fileKey = createHash('sha1').update(`${parent.file_key}#${articles[0].messageId}`).digest('hex')
|
||||
const inserted = await trx
|
||||
.knexClient('binaries')
|
||||
.insert({
|
||||
file_key: fileKey,
|
||||
poster: parent.poster,
|
||||
subject: parent.subject,
|
||||
name: parent.name,
|
||||
total_parts: parent.total_parts,
|
||||
groups: JSON.stringify(parent.groups),
|
||||
posted_at: parent.posted_at,
|
||||
size: parent.size,
|
||||
release_id: parent.release_id,
|
||||
parent_binary_id: parent.id,
|
||||
first_seen_at: now,
|
||||
last_seen_at: now,
|
||||
})
|
||||
.onConflict('file_key')
|
||||
.ignore()
|
||||
.returning('id')
|
||||
if (inserted.length === 0) continue
|
||||
const id = Number(inserted[0].id)
|
||||
await trx.knexClient('parts').insert(
|
||||
articles.map((a) => ({
|
||||
binary_id: id,
|
||||
number: a.number,
|
||||
message_id: a.messageId,
|
||||
bytes: a.bytes,
|
||||
article_number: a.articleNumber,
|
||||
pcrc32: a.pcrc32,
|
||||
part_size: a.partSize,
|
||||
file_crc32: a.fileCrc32,
|
||||
}))
|
||||
)
|
||||
// The fingerprint of a volume is that of its part 1 article.
|
||||
await trx.rawQuery(
|
||||
`UPDATE binaries SET md5_16k = (
|
||||
SELECT md5_16k FROM part_conflicts WHERE binary_id = ? AND message_id = ?
|
||||
UNION ALL SELECT md5_16k FROM parts WHERE binary_id = ? AND message_id = ? LIMIT 1)
|
||||
WHERE id = ?`,
|
||||
[parent.id, articles[0].messageId, parent.id, articles[0].messageId, id]
|
||||
)
|
||||
ids.push(id)
|
||||
}
|
||||
|
||||
await trx.from('binaries').where('id', parent.id).update({ split_at: now })
|
||||
return ids
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export default new SplitService()
|
||||
@@ -0,0 +1,172 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import indexerConfig from '#config/indexer'
|
||||
import NntpService from '#services/NntpService'
|
||||
import NzbImportService from '#services/NzbImportService'
|
||||
import { type OverviewHeader } from '#services/IngestService'
|
||||
import { parseNzb } from '#lib/nzb/index'
|
||||
import {
|
||||
moderationTarget,
|
||||
spotterId,
|
||||
decodeSpotBinary,
|
||||
parseSpotArticleHeaders,
|
||||
parseSpotHeader,
|
||||
parseSpotXml,
|
||||
verifySpotHeader,
|
||||
type SpotHeader,
|
||||
} from '#lib/spotnet/index'
|
||||
|
||||
/** Spots whose NZB is larger than this are skipped (Spotweb skips >50 KB of XML too). */
|
||||
const MAX_NZB_SEGMENTS = 20
|
||||
/** A personal dispose only counts within this long after the spot (Spotweb: 5 days). */
|
||||
const DISPOSE_WINDOW_SECONDS = 5 * 24 * 60 * 60
|
||||
/** A claimed spot whose worker died is taken again after this long. */
|
||||
const RECLAIM_AFTER = '1 hour'
|
||||
|
||||
export function isSpotGroup(name: string): boolean {
|
||||
return indexerConfig.spotGroups.includes(name)
|
||||
}
|
||||
|
||||
/**
|
||||
* Spotnet (CONCEPT.md D29). Spots are signed posts in free.pt; their full
|
||||
* header holds XML with the message ids of a compressed NZB, which is
|
||||
* imported like a posted NZB (D26). Spot NZBs name many obfuscated posts
|
||||
* (nyuu, ngPost, the skipped multi-group uploader).
|
||||
*/
|
||||
class SpotService {
|
||||
/** Queues the spots among a batch of one spot group's headers. Returns how many. */
|
||||
async recordHeaders(groupName: string, overviews: [number, OverviewHeader][]): Promise<number> {
|
||||
const now = new Date()
|
||||
const rows = []
|
||||
for (const [articleNumber, header] of overviews) {
|
||||
if (!header?.from || !header['message-id']) continue
|
||||
const spot = parseSpotHeader(header.from, header.subject ?? '', header['message-id'], header.date)
|
||||
if (!spot?.title) continue
|
||||
rows.push({
|
||||
message_id: spot.messageId,
|
||||
group_name: groupName,
|
||||
article_number: Number(articleNumber),
|
||||
title: spot.title,
|
||||
tag: spot.tag || null,
|
||||
poster: spot.poster,
|
||||
category: spot.category,
|
||||
subcats: JSON.stringify(spot.subcats),
|
||||
size: spot.size || null,
|
||||
posted_at: spot.stamp ? new Date(spot.stamp * 1000) : null,
|
||||
key_id: spot.keyId,
|
||||
spotter_id: spotterId(spot.selfSignedPubKey) || null,
|
||||
created_at: now,
|
||||
})
|
||||
}
|
||||
if (rows.length === 0) return 0
|
||||
await db.knexQuery().table('spots').insert(rows).onConflict('message_id').ignore()
|
||||
return rows.length
|
||||
}
|
||||
|
||||
/** Takes up to `limit` pending spots, oldest first; they are marked `processing`. */
|
||||
async claim(limit: number): Promise<any[]> {
|
||||
const { rows } = await db.rawQuery(
|
||||
`UPDATE spots SET status = 'processing', processed_at = now()
|
||||
WHERE message_id IN (
|
||||
SELECT message_id FROM spots
|
||||
WHERE status = 'pending' OR (status = 'processing' AND processed_at < now() - interval '${RECLAIM_AFTER}')
|
||||
ORDER BY created_at, article_number LIMIT ? FOR UPDATE SKIP LOCKED)
|
||||
RETURNING *`,
|
||||
[limit]
|
||||
)
|
||||
return rows
|
||||
}
|
||||
|
||||
/** Verifies the spot on its raw header, then reads and imports its NZB. Returns the new status. */
|
||||
async process(spot: any): Promise<string> {
|
||||
const update: Record<string, unknown> = { processed_at: new Date(), error: null }
|
||||
try {
|
||||
const raw = await this.fetch(`HEAD <${spot.message_id}>`)
|
||||
const { headers, xml } = parseSpotArticleHeaders(raw.toString('latin1'))
|
||||
// The signature covers the ISO-8859-1 bytes, which the overview (read as UTF-8) may have mangled.
|
||||
const header = parseSpotHeader(headers.from ?? '', headers.subject ?? '', spot.message_id, headers.date)
|
||||
if (!header || !verifySpotHeader(header)) {
|
||||
update.status = 'unverified'
|
||||
} else if (moderationTarget(header)) {
|
||||
update.status = 'moderation'
|
||||
await this.moderate(header, moderationTarget(header)!)
|
||||
} else {
|
||||
const parsed = parseSpotXml(xml)
|
||||
update.title = header.title
|
||||
update.poster = header.poster
|
||||
update.spotter_id = spotterId(header.selfSignedPubKey) || null
|
||||
if (parsed?.website) update.website = parsed.website
|
||||
const segments = parsed?.nzbSegments ?? []
|
||||
if (segments.length === 0 || segments.length > MAX_NZB_SEGMENTS) {
|
||||
update.status = 'no-nzb'
|
||||
} else {
|
||||
const bodies = []
|
||||
for (const id of segments) bodies.push(await this.fetch(`BODY <${id.replace(/^<|>$/g, '')}>`))
|
||||
const nzbXml = decodeSpotBinary(bodies).toString('utf8')
|
||||
const nzb = parseNzb(nzbXml)
|
||||
if (!nzb || nzb.files.length === 0) {
|
||||
update.status = 'no-nzb'
|
||||
} else {
|
||||
const groups = [...new Set(nzb.files.flatMap((f) => f.groups))]
|
||||
const result = await NzbImportService.import(
|
||||
{
|
||||
id: null,
|
||||
filename: `${header.title}.nzb`,
|
||||
poster: header.poster,
|
||||
groups,
|
||||
date: header.stamp * 1000,
|
||||
message_ids: { '1': { id: spot.message_id } },
|
||||
},
|
||||
nzbXml,
|
||||
{ name: header.title, source: 'spot', moveContainer: false }
|
||||
)
|
||||
update.status = result ? 'imported' : 'no-nzb'
|
||||
if (result) update.release_id = result.releaseId
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
update.status = 'failed'
|
||||
update.error = String(error.message).slice(0, 500)
|
||||
logger.warn(`Spot ${spot.message_id} "${spot.title}": ${error.message}`)
|
||||
}
|
||||
await db.from('spots').where('message_id', spot.message_id).update(update)
|
||||
return update.status as string
|
||||
}
|
||||
|
||||
/**
|
||||
* Applies a delete/dispose/remove: the target spot is marked `removed`
|
||||
* (its release is then hidden from search). As in Spotweb, a personal
|
||||
* dispose must come from the same spotter within 5 days; a moderator's
|
||||
* (signed with Spotnet's key, no personal key) always counts.
|
||||
*/
|
||||
private async moderate(header: SpotHeader, targetId: string) {
|
||||
const target = await db.from('spots').where('message_id', targetId).first()
|
||||
if (!target) return
|
||||
const personal = header.selfSignedPubKey.length > 50 ? spotterId(header.selfSignedPubKey) : ''
|
||||
if (personal) {
|
||||
const age = header.stamp - (target.posted_at ? new Date(target.posted_at).getTime() / 1000 : 0)
|
||||
if (target.spotter_id !== personal || age > DISPOSE_WINDOW_SECONDS) return
|
||||
}
|
||||
await db.from('spots').where('message_id', targetId).update({ status: 'removed' })
|
||||
logger.info(`Spot "${target.title}" removed by ${personal ? 'its poster' : 'a moderator'}.`)
|
||||
}
|
||||
|
||||
/** One HEAD or BODY, as raw bytes. */
|
||||
private async fetch(command: string): Promise<Buffer> {
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
const { data } = await conn._artcmd(command)
|
||||
return Buffer.from(data.buffer, data.byteOffset, data.byteLength)
|
||||
} catch (error: any) {
|
||||
// 430 (no such article) is an answer, not a broken connection.
|
||||
failed = !/^4\d\d/.test(String(error.message))
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export default new SpotService()
|
||||
@@ -1,19 +0,0 @@
|
||||
import { decode } from 'simple-yenc'
|
||||
|
||||
export class YencFile {
|
||||
private buffer: Buffer | null = null
|
||||
|
||||
public processPart(partBuffer: Buffer) {
|
||||
// This is a simplified implementation.
|
||||
// simple-yenc's decode function is synchronous and works on a full buffer.
|
||||
// A more complex implementation would handle multi-part decoding.
|
||||
this.buffer = decode(partBuffer)
|
||||
}
|
||||
|
||||
public getBuffer(): Buffer {
|
||||
if (!this.buffer) {
|
||||
throw new Error('No data has been processed yet.')
|
||||
}
|
||||
return this.buffer
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
export function parseYencMeta(buffer: Buffer): { header: Record<string, any>; crc32?: string } {
|
||||
const text = buffer.toString('latin1')
|
||||
const lines = text.split(/\\r?\\n/)
|
||||
|
||||
const header: Record<string, any> = {}
|
||||
let crc32: string | undefined
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('=ybegin')) {
|
||||
const parts = line.split(' ')
|
||||
parts.forEach((part) => {
|
||||
if (part.includes('=')) {
|
||||
const [key, value] = part.split('=')
|
||||
if (key === 'name') {
|
||||
header[key] = value.trim()
|
||||
} else {
|
||||
header[key] = parseInt(value, 10)
|
||||
}
|
||||
}
|
||||
})
|
||||
} else if (line.startsWith('=ypart')) {
|
||||
const match = /begin=(\d+)/.exec(line)
|
||||
if (match) {
|
||||
header.partBegin = parseInt(match[1], 10)
|
||||
}
|
||||
} else if (line.startsWith('=yend')) {
|
||||
const match = /crc32=([a-fA-F0-9]+)/.exec(line)
|
||||
if (match) {
|
||||
crc32 = match[1]
|
||||
}
|
||||
break // End of yenc data
|
||||
}
|
||||
}
|
||||
|
||||
return { header, crc32 }
|
||||
}
|
||||
Reference in new issue
Block a user