TruncatedReply: a compressed reply the server cut at a ".\r\n" fails in 2 s; ranges:dates samples plain
Test and deploy / test (push) Successful in 53s
Test and deploy / edge (push) Successful in 1m58s
Test and deploy / deploy (push) Successful in 1m4s

Eweka stops a compressed XOVER reply at the first ".\r\n" in its own
output (78 01 2e 0d 0a and nothing more), which some small replies begin
with; the text path waited for more until the reply timeout. ranges:dates
uses plain connections of its own instead of the pool.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Fable 5.1 committed 2026-10-25 12:49:53 +01:00
1 parent 1e01702dff
commit 3cc7d9738e
5 files changed
+188 -42

No files matched your search

+50 -24
View File
@@ -1,8 +1,9 @@
import { BaseCommand, flags } from '@adonisjs/core/ace'
import type { CommandOptions } from '@adonisjs/core/types/ace'
import db from '@adonisjs/lucid/services/db'
import env from '#start/env'
import coordinatorConfig from '#config/coordinator'
import NntpPool from '#services/nntp_pool'
import { XoverClient } from '#lib/nntp/index'
import { NntpShare } from '#services/nntp_shares'
import { estimateDate, sampleGroup, type DateSample } from '#services/group_dates'
@@ -39,13 +40,9 @@ export default class RangesDates extends BaseCommand {
declare estimateOnly: boolean
async run() {
try {
if (!this.estimateOnly) await this.sample()
await this.estimate()
await this.report()
} finally {
await NntpPool.close()
}
if (!this.estimateOnly) await this.sample()
await this.estimate()
await this.report()
}
private groups(missing = this.missing) {
@@ -62,20 +59,26 @@ export default class RangesDates extends BaseCommand {
const every = this.every ?? 500_000_000
const connections = this.connections ?? 6
const groups = await this.groups()
// Connections of its own, without compression: Eweka cuts a compressed reply at the first ".\r\n" in its
// output, which 16 of 14k small XOVERs hit (TruncatedReply; LOG.md 2026-10-25 (105)); plain, every one
// answers, and 32 lines are a few kB. Taken from the account as a share (the coordinator and the prober make
// room for them), as many as asked for.
const workers = Math.max(1, Math.min(connections, groups.length))
const share = new NntpShare(
'dates',
coordinatorConfig.nntpShares.coordinatorPriority + 1,
1,
connections
)
const first = await NntpPool.share(share)
const sharing = setInterval(
() =>
NntpPool.share(share).catch((error) => this.logger.warning(`NNTP share: ${error.message}`)),
coordinatorConfig.nntpShares.everyMs
workers,
workers
)
let open = 0
const sync = () =>
share
.sync(workers, open)
.catch((error) => this.logger.warning(`NNTP share: ${error.message}`))
await sync()
const sharing = setInterval(sync, coordinatorConfig.nntpShares.everyMs)
this.logger.info(
`Sampling ${groups.length} groups over ${first.granted} connections, a sample every ${every / 1e6}M articles.`
`Sampling ${groups.length} groups over ${workers} plain connections, a sample every ${every / 1e6}M articles.`
)
const startedAt = Date.now()
let done = 0
@@ -83,12 +86,32 @@ export default class RangesDates extends BaseCommand {
let samples = 0
const queue = [...groups]
const worker = async () => {
let client: XoverClient | null = null
const use = async <T>(work: (c: XoverClient) => Promise<T>): Promise<T> => {
if (!client) {
client = await XoverClient.connect({
host: env.get('NNTP_HOST')!,
port: env.get('NNTP_PORT', 563),
user: env.get('NNTP_USER'),
password: env.get('NNTP_PASSWORD'),
secure: env.get('NNTP_SECURE', true),
compress: false,
timeoutMs: coordinatorConfig.nntpTimeoutMs,
})
open++
}
try {
return await work(client)
} catch (error) {
client.close()
client = null
open--
throw error
}
}
for (let g = queue.shift(); g; g = queue.shift()) {
try {
const result = await sampleGroup((work) => NntpPool.use(work), g.name, {
every,
size: 32,
})
const result = await sampleGroup(use, g.name, { every, size: 32 })
for (const f of result.failed) this.logger.warning(`${g.name} at ${f}`)
await db.transaction(async (trx) => {
await trx.from('group_dates').where('group_id', g.id).delete()
@@ -116,13 +139,16 @@ export default class RangesDates extends BaseCommand {
`${done} of ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`
)
}
if (client) {
;(client as XoverClient).close()
open--
}
}
try {
await Promise.all(
Array.from({ length: Math.max(1, Math.min(connections, groups.length)) }, worker)
)
await Promise.all(Array.from({ length: workers }, worker))
} finally {
clearInterval(sharing)
await share.sync(0, 0).catch(() => {})
}
this.logger.info(
`Sampled ${groups.length} groups: ${samples} samples, ${commands} commands, ${((Date.now() - startedAt) / 1000).toFixed(0)} s.`