Rebuild: phase 2's prober reads the probe queue
probe:run (compose service prober, PROBE_CONNECTIONS, default 6) claims waiting candidates of settled releases (lone, then par2, then archive), fetches each article with BODY over connections of its own (no compression) and inspects it: the yEnc name and size, the content type from the first bytes, a par2 set's files, a RAR volume's entries and whether they're encrypted. Results per release and kind in release_inspections (CHUNK_TABLESPACE, the HDD); on the release passworded, inner_name (the largest content name) and inspected_at. 430 = missing, not yEnc = error, both final; connection trouble puts the candidate back. Fetch and parse in one process: an article takes ~0.1 s per connection, parsing a few ms (no queue in between). Parsers from voyager 1 (lib/yenc, lib/rar, lib/par2, lib/content sniff) with their tests; XoverClient.body; yencode. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
d72fe2f13f
commit
8bd6d6aa5f
30 files changed
+1666
-7
No files matched your search
+1
-1
@@ -1,4 +1,4 @@
|
||||
export { XoverClient, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js'
|
||||
export { XoverClient, MissingArticle, parseOverviews, type Overview, type XoverClientOptions } from './xover_client.js'
|
||||
export { parseXrefGroups } from './xref.js'
|
||||
export { parseGroupResponse, type GroupInfo } from './group.js'
|
||||
export { hedge, type Request, type Hedged } from './hedge.js'
|
||||
@@ -34,6 +34,9 @@ const DOT_LINE = Buffer.from('.\r\n')
|
||||
* followed by a plain "." line; an empty range is a 420 without body.
|
||||
* About six times less traffic per header (LOG.md 2026-10-05).
|
||||
*/
|
||||
/** The server doesn't have the article (430): taken down or expired. */
|
||||
export class MissingArticle extends Error {}
|
||||
|
||||
export class XoverClient {
|
||||
private chunks: Buffer[] = []
|
||||
private length = 0
|
||||
@@ -228,6 +231,37 @@ export class XoverClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* An article's body by message-id, raw (dot-stuffed, "\r\n.\r\n" cut off):
|
||||
* yEnc decoding undoes the stuffing. A connection without COMPRESS GZIP.
|
||||
* 430 (no such article) throws MissingArticle.
|
||||
*/
|
||||
async body(messageId: string): Promise<Buffer> {
|
||||
return this.timed(async () => {
|
||||
const id = messageId.startsWith('<') ? messageId : `<${messageId}>`
|
||||
const reply = await this.command(`BODY ${id}`)
|
||||
if (reply.startsWith('430') || reply.startsWith('423')) throw new MissingArticle(reply)
|
||||
await this.expect(reply, ['222'])
|
||||
return this.rawBody()
|
||||
})
|
||||
}
|
||||
|
||||
/** A multi-line body up to "\r\n.\r\n", as bytes, the terminator cut off. */
|
||||
private async rawBody(): Promise<Buffer> {
|
||||
let searched = 0
|
||||
while (true) {
|
||||
const all = this.buffer()
|
||||
if (all.length >= 3 && searched === 0 && all.subarray(0, 3).equals(DOT_LINE)) {
|
||||
this.take(3)
|
||||
return Buffer.alloc(0)
|
||||
}
|
||||
const at = all.indexOf(TERMINATOR, Math.max(0, searched - TERMINATOR.length))
|
||||
if (at !== -1) return Buffer.from(this.take(at + TERMINATOR.length).subarray(0, at + 2))
|
||||
searched = all.length
|
||||
await this.more()
|
||||
}
|
||||
}
|
||||
|
||||
/** A plain multi-line body, up to "\r\n.\r\n", dot-unstuffed. */
|
||||
private async plainBody(): Promise<string> {
|
||||
let searched = 0
|
||||
|
||||
Reference in new issue
Block a user