Fix findCounter infinite loop; corpus filter skips chunks past 256 MB
Test and deploy / test (push) Successful in 47s
Test and deploy / edge (push) Successful in 1m45s
Test and deploy / deploy (push) Successful in 2m5s

lastIndexOf('(', -1) searches from 0 again, so a subject starting with a "("
that isn't a counter, without a quote, looped forever. The corpus filter hit
it at 11:23 on Oct 24: the coordinator (and corpus:filter) spun at 100% CPU
for 10 hours and no range finished. The Rust find_counter stops at 0 already.

A chunk inflating past V8's longest string can't be filtered as text; such a
chunk is now stored unfiltered instead of throwing.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Daan MeijerandClaude Opus 5.5 committed 2026-10-24 21:08:22 +02:00
1 parent 2dd5b4cd81
commit 03ca96b255
3 files changed
+10 -1

No files matched your search

+5 -1
View File
@@ -9,6 +9,8 @@ import type { ChunkSpan } from '#services/spool'
const inflate = promisify(unzip)
const deflate = promisify(deflateCb)
const gzip = promisify(gzipCb)
/** The largest inflated chunk the filter reads as one string (V8's limit is 0x1fffffe8 characters). */
const MAX_FILTER_BYTES = 256 * 1024 * 1024
/** What a filtered chunk was filtered with (CORPUS_FILTER): the rule set and the group priorities of its lease. */
export interface CorpusFilter {
@@ -91,7 +93,9 @@ class Corpus {
const file = this.path(range, span)
await mkdir(dir, { recursive: true })
let stored = data
if (filter) {
// Past V8's longest string (0x1fffffe8) the text can't be filtered; such a chunk is stored as it came (the
// worker drops the same lines itself). Seen once: a chunk inflating past 512 MB (LOG.md 2026-10-24 (102)).
if (filter && raw.length < MAX_FILTER_BYTES) {
// Latin-1 both ways: every byte back as it was.
const { kept, drops } = filterOverviews(raw.toString('latin1'), range.group, filter.priorities, filter.ruleVersion)
stored = await deflate(Buffer.from(kept, 'latin1'), { level: 9 })
+2
View File
@@ -47,6 +47,8 @@ interface Counter {
function findCounter(subject: string, after: number): Counter | null {
let open = subject.length
while (true) {
// lastIndexOf('(', -1) searches from 0 again: a "(" at the start that isn't a counter looped forever.
if (open === 0) return null
open = subject.lastIndexOf('(', open - 1)
if (open <= after) return null
let i = open + 1
+3
View File
@@ -37,5 +37,8 @@ test.group('edge / scanOverviewText and SegmentScanner', () => {
// Obfuscated, or a counter only before the name: no segment.
assert.isNull(scanner.segment(9, 'Xq8r2KdLm3Zw91PqA7zz', 'p', '', '<o@x>', '1', xref, 'alt.binaries.test'))
assert.isNull(scanner.segment(9, '(1/2) "x.bin" yEnc', 'p', '', '<o@x>', '1', xref, 'alt.binaries.test'))
// A "(" at the start that isn't a counter, no quote: no segment (it looped forever, LOG.md 2026-10-24 (102)).
assert.isNull(scanner.segment(9, '(Some Show) S01E01 x264', 'p', '', '<o@x>', '1', xref, 'alt.binaries.test'))
assert.equal(scanner.segment(9, '(Some Show) S01E01.mkv (3/9)', 'p', '', '<o@x>', '1', xref, 'alt.binaries.test')?.number, 3)
})
})