From f642a49d7da0e155ba1c5025e20fa1a63598441c Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Fri, 9 Oct 2026 17:45:00 +0200 Subject: [PATCH] Rebuild: post a batch early when 100k files wait; log failing stats refreshes A picture group's 500k articles made batches over the server's 256 MB body limit: HTTP 413, the range given back and leased again in a loop (148597: 39 times in 30 minutes). The worker now also posts once 100,000 completed or edge-cut files wait (RangeIndexer.pending). A stats refresh that failed kept the previous totals silently; it logs the error now. Co-Authored-By: Claude Opus 5.5 --- app/services/search_service.ts | 2 ++ commands/edge_node.ts | 5 ++++- lib/edge/range_indexer.ts | 7 +++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/app/services/search_service.ts b/app/services/search_service.ts index 63b4c8f..ef3e34a 100644 --- a/app/services/search_service.ts +++ b/app/services/search_service.ts @@ -1,4 +1,5 @@ import db from '@adonisjs/lucid/services/db' +import logger from '@adonisjs/core/services/logger' import NzbService from '#services/nzb_service' export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name' @@ -192,6 +193,7 @@ class SearchService { this.statsCache = { at: Date.now(), value } }) .catch((error) => { + logger.warn(`Stats: ${error.message}`) if (!this.statsCache) throw error }) .finally(() => { diff --git a/commands/edge_node.ts b/commands/edge_node.ts index 5040534..035353a 100644 --- a/commands/edge_node.ts +++ b/commands/edge_node.ts @@ -8,6 +8,8 @@ import { gzip, unzip } from 'node:zlib' import { RangeIndexer, decodeFrames, isKeepalive, throttle, toWire, type EdgeBatch, type Lease } from '#lib/edge/index' const IDLE_MS = 30_000 +/** A batch is posted once this many files (completed, or cut at an edge) wait, whatever its articles. */ +const MAX_PENDING_FILES = 100_000 const unzipAsync = promisify(unzip) const gzipAsync = promisify(gzip) const RETRIES = 8 @@ -166,7 +168,8 @@ export default class EdgeNode extends BaseCommand { indexer.addText(text.toString('utf8'), chunk.to + 1) reachedEnd = chunk.to >= range.end if (reachedEnd) indexer.finish() - if (reachedEnd || chunk.to + 1 - batchFrom >= this.batchArticles) { + // Also early when many files wait: a picture group's 500k articles made batches over the server's 256 MB limit (HTTP 413). + if (reachedEnd || chunk.to + 1 - batchFrom >= this.batchArticles || indexer.pending >= MAX_PENDING_FILES) { post(indexer.take()) batchFrom = chunk.to + 1 } diff --git a/lib/edge/range_indexer.ts b/lib/edge/range_indexer.ts index 56cfe8a..7934fde 100644 --- a/lib/edge/range_indexer.ts +++ b/lib/edge/range_indexer.ts @@ -188,6 +188,13 @@ export class RangeIndexer { return this.aggregator.openFiles } + /** Files completed and fragments cut since the last take: how big the next batch is. */ + get pending(): number { + let files = this.batch.fragments.length + for (const list of this.completed.values()) files += list.length + return files + } + /** Completed files, gathered per release until the batch is taken: one chunk per release per batch. */ private completed = new Map[]>()