Daan MeijerandClaude Opus 5.5
2dbca9b152
Inspection is split into prepare (database), probe (lib/content/probe: downloads, decoding and parsing, no database) and apply (database). worker:collection runs all three; seed:node with SEED_INSPECTORS leases jobs from the same queues through the seed API (BullMQ manual leases), fetches articles through the seed API and posts the probes back. A node now needs only SEED_URL and SEED_TOKEN: the settings the app requires but a node never uses get placeholders for seed:node. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
83 lines
3.1 KiB
TypeScript
83 lines
3.1 KiB
TypeScript
import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '../yenc/index.js'
|
|
import type { VolumeReader } from './walk.js'
|
|
|
|
/** Downloads one article body (on a pooled connection, or through the seed API's relay). */
|
|
export type BodyFetcher = (messageId: string) => Promise<Buffer>
|
|
|
|
/**
|
|
* Reads volumes (each a list of segment message ids, in order) a segment at
|
|
* a time, finding the segment for an offset from the part size (yEnc
|
|
* posters use one size per set) and correcting from the `=ypart` offsets
|
|
* when a guess is off. Stops reading after `maxSegments` downloads.
|
|
*/
|
|
export class SegmentReader implements VolumeReader {
|
|
fetched = 0
|
|
private partSize = 0
|
|
private readonly sizes = new Map<number, number>()
|
|
private readonly cache = new Map<string, DecodedSegment>()
|
|
|
|
constructor(
|
|
private readonly files: { id: string }[][],
|
|
private readonly maxSegments: number,
|
|
private readonly fetchBody: BodyFetcher
|
|
) {}
|
|
|
|
get volumes() {
|
|
return this.files.length
|
|
}
|
|
|
|
size(v: number) {
|
|
return this.sizes.get(v) ?? null
|
|
}
|
|
|
|
async read(v: number, offset: number, more: boolean): Promise<Buffer | null> {
|
|
const count = this.files[v].length
|
|
let index = this.partSize ? Math.min(count - 1, Math.floor(offset / this.partSize)) : 0
|
|
for (let tries = 0; tries < 4; tries++) {
|
|
const segment = await this.segment(v, index)
|
|
if (!segment) return null
|
|
const end = segment.offset + segment.data.length
|
|
if (offset >= segment.offset && offset < end) {
|
|
const data = segment.data.subarray(offset - segment.offset)
|
|
if (!more || index + 1 >= count) return data
|
|
const next = await this.segment(v, index + 1)
|
|
return next ? Buffer.concat([data, next.data]) : null
|
|
}
|
|
const step = Math.floor((offset - segment.offset) / Math.max(1, segment.data.length))
|
|
const moved = Math.max(0, Math.min(count - 1, index + (step === 0 ? (offset < segment.offset ? -1 : 1) : step)))
|
|
if (moved === index) return null
|
|
index = moved
|
|
}
|
|
return null
|
|
}
|
|
|
|
private async segment(v: number, index: number): Promise<DecodedSegment | null> {
|
|
const key = `${v}:${index}`
|
|
const cached = this.cache.get(key)
|
|
if (cached) return cached
|
|
const part = this.files[v][index]
|
|
if (!part?.id || this.fetched >= this.maxSegments) return null
|
|
this.fetched++
|
|
let body: Buffer
|
|
try {
|
|
body = await this.fetchBody(part.id)
|
|
} catch {
|
|
// A missing article (or the relay failing) ends the walk with what was found.
|
|
return null
|
|
}
|
|
let segment: DecodedSegment
|
|
try {
|
|
segment = decodeSegment(body)
|
|
} catch (error) {
|
|
if (!(error instanceof YencError) || error instanceof YencCrcError) return null
|
|
segment = decodeSegment(body, { lenientSize: true })
|
|
}
|
|
this.sizes.set(v, segment.meta.begin.size)
|
|
if (!this.partSize && segment.meta.part) this.partSize = segment.meta.part.end - segment.meta.part.begin + 1
|
|
// Walks move forward: the last few segments are enough.
|
|
if (this.cache.size >= 4) this.cache.delete(this.cache.keys().next().value!)
|
|
this.cache.set(key, segment)
|
|
return segment
|
|
}
|
|
}
|