Content inspection and archive walks on seeding nodes (relay)
Inspection is split into prepare (database), probe (lib/content/probe: downloads, decoding and parsing, no database) and apply (database). worker:collection runs all three; seed:node with SEED_INSPECTORS leases jobs from the same queues through the seed API (BullMQ manual leases), fetches articles through the seed API and posts the probes back. A node now needs only SEED_URL and SEED_TOKEN: the settings the app requires but a node never uses get placeholders for seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
51975fe603
commit
2dbca9b152
22 files changed
+1198
-540
No files matched your search
@@ -1,15 +1,10 @@
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import type { Segment } from '#lib/nzb/index'
|
||||
import logger from '@adonisjs/core/services/logger'
|
||||
import NntpService from '#services/NntpService'
|
||||
import IngestService, { PAR2_MATCH_WINDOW_SECONDS } from '#services/IngestService'
|
||||
import { decodeSegment, YencFile } from '#lib/yenc/index'
|
||||
import { isPar2, parsePar2, type Par2Set } from '#lib/par2/index'
|
||||
import type { Par2Set } from '#lib/par2/index'
|
||||
import { par2ReleaseIdentity, par2ReleaseName } from '#lib/release/index'
|
||||
import NameService, { WEAK_RELEASE_SQL, isBetterName } from '#services/NameService'
|
||||
|
||||
/** Par2 index files are small; recovery volumes can be large and are not needed. */
|
||||
const MAX_PARTS = 20
|
||||
/** Size-only matching below this is too ambiguous. */
|
||||
const SIZE_ONLY_MIN = 1024 * 1024
|
||||
|
||||
@@ -32,34 +27,6 @@ export interface Par2Result {
|
||||
* body-derived binaries carry their exact size. See CONCEPT.md (D16).
|
||||
*/
|
||||
class Par2Service {
|
||||
/**
|
||||
* Reads a par2 file whose first segment is already decoded, downloading the
|
||||
* remaining parts if there are any. Returns null when it is not par2.
|
||||
*/
|
||||
async read(first: Buffer, segments: Segment[]): Promise<Par2Set | null> {
|
||||
if (!isPar2(first)) return null
|
||||
const ids = segments.map((part) => part.id)
|
||||
if (ids.length > MAX_PARTS) return null
|
||||
if (ids.length === 1) return parsePar2(first)
|
||||
|
||||
let assembled: YencFile | undefined
|
||||
const conn = await NntpService.acquire()
|
||||
let failed = false
|
||||
try {
|
||||
for (const id of ids) {
|
||||
const segment = decodeSegment(await NntpService.fetchBody(conn, id))
|
||||
assembled ??= YencFile.fromSegment(segment)
|
||||
assembled.add(segment)
|
||||
}
|
||||
} catch (error) {
|
||||
failed = true
|
||||
throw error
|
||||
} finally {
|
||||
NntpService.release(conn, failed)
|
||||
}
|
||||
return parsePar2(assembled!.getBuffer())
|
||||
}
|
||||
|
||||
async attach(binary: any, set: Par2Set): Promise<Par2Result> {
|
||||
const result = await db.transaction(async (trx) => {
|
||||
// Exact: size and first-16-KiB MD5 of a listed file. Only these may pull
|
||||
|
||||
Reference in new issue
Block a user