From 2bbabc08de433666669692d6a25e0a9347132d5d Mon Sep 17 00:00:00 2001 From: Daan Meijer Date: Tue, 13 Oct 2026 05:55:26 +0200 Subject: [PATCH] corpus_sweep capture: budget for the marked ranges still filling Only the bytes on disk were checked: at 296.6 GB with 2,300 ranges complete it marked 100 more (~310 GB). The marked ranges not complete count at the stored ranges' mean size, and no more are marked than fit under 300 GB. Co-Authored-By: Claude Opus 5.5 --- deploy/corpus_sweep.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/deploy/corpus_sweep.py b/deploy/corpus_sweep.py index ae391e8..43e6ea4 100644 --- a/deploy/corpus_sweep.py +++ b/deploy/corpus_sweep.py @@ -55,8 +55,16 @@ def corpus_bytes(): def capture(n): used = corpus_bytes() - if used >= MAX_BYTES: - print(f'corpus at {used / 1e9:.0f} GB of {MAX_BYTES / 1e9:.0f}: nothing more marked') + # Marked ranges not complete yet take room too: at the stored ranges' mean size, less what they hold already + # (2026-10-13: 2,300 complete at 296.6 GB and 100 more marked would have made ~310 GB). + complete = sum(1 for _ in complete_ranges()) + marked = int(psql('SELECT count(*) FROM ranges WHERE capture;')[0][0]) + mean = used / complete if complete else 0 + pending = max(0, marked - complete) * mean + room = MAX_BYTES - max(used, complete * mean + pending) + n = min(n, int(room // mean)) if mean else n + if n <= 0: + print(f'corpus at {used / 1e9:.1f} GB of {MAX_BYTES / 1e9:.0f} (with {marked - complete} ranges still filling, ~{(complete * mean + pending) / 1e9:.0f} GB): nothing more marked') return top = [line.split('\t')[1] for line in open(os.path.join(ROOT, 'groups.tsv')).read().splitlines()[:TOP]] names = ','.join("'" + g.replace("'", "''") + "'" for g in top)