Fix findCounter infinite loop; corpus filter skips chunks past 256 MB
lastIndexOf('(', -1) searches from 0 again, so a subject starting with a "("
that isn't a counter, without a quote, looped forever. The corpus filter hit
it at 11:23 on Oct 24: the coordinator (and corpus:filter) spun at 100% CPU
for 10 hours and no range finished. The Rust find_counter stops at 0 already.
A chunk inflating past V8's longest string can't be filtered as text; such a
chunk is now stored unfiltered instead of throwing.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2dd5b4cd81
commit
03ca96b255
3 files changed
+10
-1
No files matched your search
@@ -9,6 +9,8 @@ import type { ChunkSpan } from '#services/spool'
|
||||
const inflate = promisify(unzip)
|
||||
const deflate = promisify(deflateCb)
|
||||
const gzip = promisify(gzipCb)
|
||||
/** The largest inflated chunk the filter reads as one string (V8's limit is 0x1fffffe8 characters). */
|
||||
const MAX_FILTER_BYTES = 256 * 1024 * 1024
|
||||
|
||||
/** What a filtered chunk was filtered with (CORPUS_FILTER): the rule set and the group priorities of its lease. */
|
||||
export interface CorpusFilter {
|
||||
@@ -91,7 +93,9 @@ class Corpus {
|
||||
const file = this.path(range, span)
|
||||
await mkdir(dir, { recursive: true })
|
||||
let stored = data
|
||||
if (filter) {
|
||||
// Past V8's longest string (0x1fffffe8) the text can't be filtered; such a chunk is stored as it came (the
|
||||
// worker drops the same lines itself). Seen once: a chunk inflating past 512 MB (LOG.md 2026-10-24 (102)).
|
||||
if (filter && raw.length < MAX_FILTER_BYTES) {
|
||||
// Latin-1 both ways: every byte back as it was.
|
||||
const { kept, drops } = filterOverviews(raw.toString('latin1'), range.group, filter.priorities, filter.ruleVersion)
|
||||
stored = await deflate(Buffer.from(kept, 'latin1'), { level: 9 })
|
||||
|
||||
Reference in new issue
Block a user