TruncatedReply: a compressed reply the server cut at a ".\r\n" fails in 2 s; ranges:dates samples plain
Eweka stops a compressed XOVER reply at the first ".\r\n" in its own output (78 01 2e 0d 0a and nothing more), which some small replies begin with; the text path waited for more until the reply timeout. ranges:dates uses plain connections of its own instead of the pool. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
1e01702dff
commit
3cc7d9738e
5 files changed
+188
-42
No files matched your search
+50
-24
@@ -1,8 +1,9 @@
|
||||
import { BaseCommand, flags } from '@adonisjs/core/ace'
|
||||
import type { CommandOptions } from '@adonisjs/core/types/ace'
|
||||
import db from '@adonisjs/lucid/services/db'
|
||||
import env from '#start/env'
|
||||
import coordinatorConfig from '#config/coordinator'
|
||||
import NntpPool from '#services/nntp_pool'
|
||||
import { XoverClient } from '#lib/nntp/index'
|
||||
import { NntpShare } from '#services/nntp_shares'
|
||||
import { estimateDate, sampleGroup, type DateSample } from '#services/group_dates'
|
||||
|
||||
@@ -39,13 +40,9 @@ export default class RangesDates extends BaseCommand {
|
||||
declare estimateOnly: boolean
|
||||
|
||||
async run() {
|
||||
try {
|
||||
if (!this.estimateOnly) await this.sample()
|
||||
await this.estimate()
|
||||
await this.report()
|
||||
} finally {
|
||||
await NntpPool.close()
|
||||
}
|
||||
if (!this.estimateOnly) await this.sample()
|
||||
await this.estimate()
|
||||
await this.report()
|
||||
}
|
||||
|
||||
private groups(missing = this.missing) {
|
||||
@@ -62,20 +59,26 @@ export default class RangesDates extends BaseCommand {
|
||||
const every = this.every ?? 500_000_000
|
||||
const connections = this.connections ?? 6
|
||||
const groups = await this.groups()
|
||||
// Connections of its own, without compression: Eweka cuts a compressed reply at the first ".\r\n" in its
|
||||
// output, which 16 of 14k small XOVERs hit (TruncatedReply; LOG.md 2026-10-25 (105)); plain, every one
|
||||
// answers, and 32 lines are a few kB. Taken from the account as a share (the coordinator and the prober make
|
||||
// room for them), as many as asked for.
|
||||
const workers = Math.max(1, Math.min(connections, groups.length))
|
||||
const share = new NntpShare(
|
||||
'dates',
|
||||
coordinatorConfig.nntpShares.coordinatorPriority + 1,
|
||||
1,
|
||||
connections
|
||||
)
|
||||
const first = await NntpPool.share(share)
|
||||
const sharing = setInterval(
|
||||
() =>
|
||||
NntpPool.share(share).catch((error) => this.logger.warning(`NNTP share: ${error.message}`)),
|
||||
coordinatorConfig.nntpShares.everyMs
|
||||
workers,
|
||||
workers
|
||||
)
|
||||
let open = 0
|
||||
const sync = () =>
|
||||
share
|
||||
.sync(workers, open)
|
||||
.catch((error) => this.logger.warning(`NNTP share: ${error.message}`))
|
||||
await sync()
|
||||
const sharing = setInterval(sync, coordinatorConfig.nntpShares.everyMs)
|
||||
this.logger.info(
|
||||
`Sampling ${groups.length} groups over ${first.granted} connections, a sample every ${every / 1e6}M articles.`
|
||||
`Sampling ${groups.length} groups over ${workers} plain connections, a sample every ${every / 1e6}M articles.`
|
||||
)
|
||||
const startedAt = Date.now()
|
||||
let done = 0
|
||||
@@ -83,12 +86,32 @@ export default class RangesDates extends BaseCommand {
|
||||
let samples = 0
|
||||
const queue = [...groups]
|
||||
const worker = async () => {
|
||||
let client: XoverClient | null = null
|
||||
const use = async <T>(work: (c: XoverClient) => Promise<T>): Promise<T> => {
|
||||
if (!client) {
|
||||
client = await XoverClient.connect({
|
||||
host: env.get('NNTP_HOST')!,
|
||||
port: env.get('NNTP_PORT', 563),
|
||||
user: env.get('NNTP_USER'),
|
||||
password: env.get('NNTP_PASSWORD'),
|
||||
secure: env.get('NNTP_SECURE', true),
|
||||
compress: false,
|
||||
timeoutMs: coordinatorConfig.nntpTimeoutMs,
|
||||
})
|
||||
open++
|
||||
}
|
||||
try {
|
||||
return await work(client)
|
||||
} catch (error) {
|
||||
client.close()
|
||||
client = null
|
||||
open--
|
||||
throw error
|
||||
}
|
||||
}
|
||||
for (let g = queue.shift(); g; g = queue.shift()) {
|
||||
try {
|
||||
const result = await sampleGroup((work) => NntpPool.use(work), g.name, {
|
||||
every,
|
||||
size: 32,
|
||||
})
|
||||
const result = await sampleGroup(use, g.name, { every, size: 32 })
|
||||
for (const f of result.failed) this.logger.warning(`${g.name} at ${f}`)
|
||||
await db.transaction(async (trx) => {
|
||||
await trx.from('group_dates').where('group_id', g.id).delete()
|
||||
@@ -116,13 +139,16 @@ export default class RangesDates extends BaseCommand {
|
||||
`${done} of ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||||
)
|
||||
}
|
||||
if (client) {
|
||||
;(client as XoverClient).close()
|
||||
open--
|
||||
}
|
||||
}
|
||||
try {
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.max(1, Math.min(connections, groups.length)) }, worker)
|
||||
)
|
||||
await Promise.all(Array.from({ length: workers }, worker))
|
||||
} finally {
|
||||
clearInterval(sharing)
|
||||
await share.sync(0, 0).catch(() => {})
|
||||
}
|
||||
this.logger.info(
|
||||
`Sampled ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
|
||||
|
||||
Reference in new issue
Block a user