Rebuild: post a batch early when 100k files wait; log failing stats refreshes

A picture group's 500k articles made batches over the server's 256 MB
body limit: HTTP 413, the range given back and leased again in a loop
(148597: 39 times in 30 minutes). The worker now also posts once
100,000 completed or edge-cut files wait (RangeIndexer.pending).
A stats refresh that failed kept the previous totals silently; it
logs the error now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

app/services/search_service.ts

@@ -1,4 +1,5 @@
 import db from '@adonisjs/lucid/services/db'
+import logger from '@adonisjs/core/services/logger'
 import NzbService from '#services/nzb_service'
 
 export type SortOrder = 'newest' | 'oldest' | 'largest' | 'files' | 'name'
@@ -192,6 +193,7 @@ class SearchService {
           this.statsCache = { at: Date.now(), value }
         })
         .catch((error) => {
+          logger.warn(`Stats: ${error.message}`)
           if (!this.statsCache) throw error
         })
         .finally(() => {

commands/edge_node.ts

@@ -8,6 +8,8 @@ import { gzip, unzip } from 'node:zlib'
 import { RangeIndexer, decodeFrames, isKeepalive, throttle, toWire, type EdgeBatch, type Lease } from '#lib/edge/index'
 
 const IDLE_MS = 30_000
+/** A batch is posted once this many files (completed, or cut at an edge) wait, whatever its articles. */
+const MAX_PENDING_FILES = 100_000
 const unzipAsync = promisify(unzip)
 const gzipAsync = promisify(gzip)
 const RETRIES = 8
@@ -166,7 +168,8 @@ export default class EdgeNode extends BaseCommand {
       indexer.addText(text.toString('utf8'), chunk.to + 1)
       reachedEnd = chunk.to >= range.end
       if (reachedEnd) indexer.finish()
-      if (reachedEnd || chunk.to + 1 - batchFrom >= this.batchArticles) {
+      // Also early when many files wait: a picture group's 500k articles made batches over the server's 256 MB limit (HTTP 413).
+      if (reachedEnd || chunk.to + 1 - batchFrom >= this.batchArticles || indexer.pending >= MAX_PENDING_FILES) {
         post(indexer.take())
         batchFrom = chunk.to + 1
       }

lib/edge/range_indexer.ts

@@ -188,6 +188,13 @@ export class RangeIndexer {
     return this.aggregator.openFiles
   }
 
+  /** Files completed and fragments cut since the last take: how big the next batch is. */
+  get pending(): number {
+    let files = this.batch.fragments.length
+    for (const list of this.completed.values()) files += list.length
+    return files
+  }
+
   /** Completed files, gathered per release until the batch is taken: one chunk per release per batch. */
   private completed = new Map<string, AggregatedFile<EdgeSegment>[]>()