Rebuild: phase 2's prober reads the probe queue
probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims waiting candidates of settled releases (lone, then par2, then archive), fetches each article with BODY over connections of its own (no compression) and inspects it: the yEnc name and size, the content type from the first bytes, a par2 set's files, a RAR volume's entries and whether they're encrypted. Results per release and kind in release_inspections (CHUNK_TABLESPACE, the HDD); on the release passworded, inner_name (the largest content name) and inspected_at. 430 = missing, not yEnc = error, both final; connection trouble puts the candidate back. Fetch and parse in one process: an article takes ~0.1 s per connection, parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc, lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body; yencode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d72fe2f13f
commit
8bd6d6aa5f
30 files changed
+1666
-7
No files matched your search
@@ -0,0 +1 @@
|
||||
export { inspect, innerName, MAX_NAMES, type Inspection } from './inspect.js'
|
||||
@@ -0,0 +1,79 @@
|
||||
import { decodeSegment, YencCrcError, YencError, type DecodedSegment } from '#lib/yenc/index'
|
||||
import { isPar2, parsePar2 } from '#lib/par2/index'
|
||||
import { parseRarHeaders } from '#lib/rar/index'
|
||||
import { sniffContentType, type ContentType } from '#lib/content/index'
|
||||
|
||||
/** What one probed article showed (REBUILD.md, part 6; stored per release and kind). */
|
||||
export interface Inspection {
|
||||
/** The file name in the yEnc header. */
|
||||
yencName: string
|
||||
/** The whole file's size, from the yEnc header. */
|
||||
size: number
|
||||
type: ContentType | null
|
||||
/** Names inside: an archive's entries or a par2 set's files, the largest first, at most MAX_NAMES. */
|
||||
names?: string[]
|
||||
/** How many entries or files there are in all. */
|
||||
entries?: number
|
||||
/** An archive with encrypted entries or headers. */
|
||||
passworded?: boolean
|
||||
}
|
||||
|
||||
/** Names kept per inspection: the catalog wants the content's name, not every volume's. */
|
||||
export const MAX_NAMES = 8
|
||||
const MAX_NAME_LENGTH = 200
|
||||
|
||||
/** Archive volumes, par2 volumes and split parts: names of the packaging, not the content. */
|
||||
const PACKAGING = /\.(rar|r\d{2,3}|s\d{2}|\d{3}|7z(\.\d+)?|zip|z\d{2}|par2|sfv|nfo|nzb)$/i
|
||||
|
||||
/** Decodes a body; some tools declare a size that leaves out a header, the CRC still has to match. */
|
||||
function decodeBody(body: Buffer): DecodedSegment {
|
||||
try {
|
||||
return decodeSegment(body)
|
||||
} catch (error) {
|
||||
if (!(error instanceof YencError) || error instanceof YencCrcError) throw error
|
||||
return decodeSegment(body, { lenientSize: true })
|
||||
}
|
||||
}
|
||||
|
||||
function largestFirst(items: { name: string; size: number }[]): string[] {
|
||||
return items
|
||||
.sort((a, b) => b.size - a.size)
|
||||
.slice(0, MAX_NAMES)
|
||||
.map((i) => i.name.slice(0, MAX_NAME_LENGTH))
|
||||
}
|
||||
|
||||
/** One article's body: its yEnc header, what the first bytes are, and for par2 and RAR what's inside. Throws when it isn't yEnc. */
|
||||
export function inspect(body: Buffer): Inspection {
|
||||
const segment = decodeBody(body)
|
||||
const result: Inspection = { yencName: segment.meta.begin.name.slice(0, MAX_NAME_LENGTH), size: segment.meta.begin.size, type: sniffContentType(segment.data) }
|
||||
if (isPar2(segment.data)) {
|
||||
// An index file fits one article; what a larger one cut off is skipped by the parser.
|
||||
const set = parsePar2(segment.data)
|
||||
if (set) {
|
||||
result.entries = set.files.length
|
||||
result.names = largestFirst(set.files.map((f) => ({ name: f.name, size: f.length })))
|
||||
}
|
||||
return result
|
||||
}
|
||||
if (result.type === 'rar4' || result.type === 'rar5') {
|
||||
const info = parseRarHeaders(segment.data)
|
||||
if (info) {
|
||||
const files = info.entries.filter((e) => !e.directory)
|
||||
result.entries = files.length
|
||||
result.names = largestFirst(files.map((e) => ({ name: e.name, size: Number(e.size ?? 0) })))
|
||||
result.passworded = info.headersEncrypted || files.some((e) => e.encrypted)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
/**
|
||||
* A release's best inner name from its inspections: the largest named entry
|
||||
* of its archive, else the largest file of its par2 set that isn't
|
||||
* packaging, else a lone file's yEnc name. Null when none says more.
|
||||
*/
|
||||
export function innerName(inspections: Partial<Record<string, Inspection>>): string | null {
|
||||
const content = (i?: Inspection) => i?.names?.find((n) => !PACKAGING.test(n))
|
||||
const pick = content(inspections.archive) ?? content(inspections.par2) ?? (inspections.lone?.yencName || undefined)
|
||||
return pick ? pick.replace(/^.*[\\/]/, '').slice(0, 255) : null
|
||||
}
|
||||
Reference in new issue
Block a user