fix(learn): budget the digest's rendered loop section, retry Codex overflows

The loop cap counted loops (max_tokens // 400), assuming a ~100-token
line. _build_digest also takes caller-supplied loops, and nothing bounded
their rendered text: 25 Grep loops with 2,000-char patterns rendered
54,080 chars against the 40,000-char budget of a 10k-token retry. The
event-stream check comes after the section, so it could not repair that.

format_loops_for_digest now takes max_chars and stops at the first loop
whose line would pass it, so the costliest loops are kept. Each line
shows its sample input cut to 120 chars (the cut detect_loops already
makes) and its first 20 message indices; the caller's LoopPatterns are
not touched. _build_digest gives the section a quarter of the char
budget, less what the header and the prior learned patterns leave short.
The prior patterns are never cut: a re-emitted section replaces its prior
one wholesale, so a partial baseline would drop bullets from the file.

_is_prompt_too_long also matches Codex's "Input exceeds the maximum
length" refusal, so a Codex overflow gets the smaller-digest retry.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-10-19 23:29:23 +02:00
co-authored by Claude Opus 5.5
parent f52a43566a
commit d9391dab3a
4 changed files with 108 additions and 50 deletions
+27 -30
View File
@@ -23,7 +23,6 @@ import subprocess
import threading
import time
import typing
from dataclasses import replace
from headroom._subprocess import Popen, run
@@ -53,13 +52,6 @@ _MAX_DIGEST_TOKENS = 80_000 # Budget for the digest (leave room for prompt + ou
# budget halves on each "prompt is too long" failure; below this the digest
# carries too little signal to produce useful recommendations.
_MIN_DIGEST_TOKENS = 10_000
# The loop section precedes the budgeted event stream, so it is bounded on its
# own: one loop per this many digest tokens (a rendered loop line is ~100
# tokens, so the section stays under a quarter of the budget and halves with it
# on a retry), each listing its first _MAX_LOOP_INDICES message indices beside
# its repeat count.
_DIGEST_TOKENS_PER_LOOP = 400
_MAX_LOOP_INDICES = 20
# CLI tools to try when no API key is set (checked in order).
# Each entry: (binary_name, model_identifier, command_prefix). The claude-cli
@@ -344,8 +336,9 @@ def _is_prompt_too_long(exc: Exception) -> bool:
Matches the known phrasings across backends: claude CLI ("Prompt is too
long"), Anthropic via LiteLLM ("prompt is too long: N tokens > M maximum"),
OpenAI ("context_length_exceeded" / "maximum context length"), and Gemini
("input token count exceeds the maximum").
OpenAI ("context_length_exceeded" / "maximum context length"), Gemini
("input token count exceeds the maximum"), and the codex CLI ("Input exceeds
the maximum length of 1048576 characters").
"""
msg = str(exc).lower()
return any(
@@ -356,6 +349,7 @@ def _is_prompt_too_long(exc: Exception) -> bool:
"maximum context length",
"context window",
"token count exceeds",
"input exceeds the maximum length",
)
)
@@ -400,29 +394,32 @@ def _build_digest(
lines.append(f"Tokens used: {total_tokens_in:,} in / {total_tokens_out:,} out")
lines.append("")
# Detected loops first — the most expensive waste pattern, so the LLM sees
# it before the (budget-truncatable) per-session event stream. Only the
# costliest fit (detect_loops sorts by waste): rendered whole, every loop
# with every index, the section alone outgrew the digest on a machine with
# years of Codex sessions, which learn scans as one project.
loop_section = format_loops_for_digest(
[
replace(lp, msg_indices=lp.msg_indices[:_MAX_LOOP_INDICES])
for lp in loops[: max_tokens // _DIGEST_TOKENS_PER_LOOP]
]
)
if loop_section:
lines.append(loop_section)
# Prior learned patterns (if any) — gives the LLM the current baseline so
# it can produce complete updated sections instead of condensed deltas.
prior_section = _build_prior_patterns_section(project)
if prior_section:
lines.append(prior_section)
# Budget tracking — stop adding events when we approach the limit
# Rough estimate: 4 chars per token
char_budget = max_tokens * 4
# Prior learned patterns (if any) — gives the LLM the current baseline so
# it can produce complete updated sections instead of condensed deltas.
# Never cut: a re-emitted section replaces its prior one wholesale, so a
# partial baseline would drop the missing bullets from the written file.
prior_section = _build_prior_patterns_section(project)
# Detected loops first — the most expensive waste pattern, so the LLM sees
# it before the (budget-truncatable) per-session event stream. The event
# stream cannot trim a section placed ahead of it, so the loops get their
# own cap: a quarter of the budget, less whatever the header and prior
# patterns leave short. Rendered whole, every loop with every index, the
# section alone outgrew the digest on a machine with years of Codex
# sessions, which learn scans as one project.
pre_event_chars = sum(len(ln) for ln in lines) + len(prior_section)
loop_section = format_loops_for_digest(
loops, max_chars=min(char_budget // 4, char_budget - pre_event_chars)
)
if loop_section:
lines.append(loop_section)
if prior_section:
lines.append(prior_section)
chars_used = sum(len(ln) for ln in lines)
for session in sessions:
+26 -10
View File
@@ -69,6 +69,11 @@ _PAGINATION_RE = re.compile("|".join(_PAGINATION_PATTERNS), re.IGNORECASE)
# otherwise identical commands do not split a loop into singletons.
_INT_RE = re.compile(r"\b\d+\b")
# A loop's digest line shows its sample input cut to this many chars and its
# first _DIGEST_MAX_INDICES message indices beside its full repeat count.
_SAMPLE_INPUT_CHARS = 120
_DIGEST_MAX_INDICES = 20
@dataclass
class LoopPattern:
@@ -290,7 +295,7 @@ def detect_loops(
LoopPattern(
tool=calls[0].name,
signature=_fuzzy_signature(signatures[key]),
sample_input=calls[0].input_summary[:120],
sample_input=calls[0].input_summary[:_SAMPLE_INPUT_CHARS],
count=count,
is_error_loop=is_error_loop,
wasted_tokens=wasted,
@@ -302,15 +307,19 @@ def detect_loops(
return loops
def format_loops_for_digest(loops: list[LoopPattern]) -> str:
def format_loops_for_digest(loops: list[LoopPattern], max_chars: int | None = None) -> str:
"""Render detected loops as a high-priority digest section for the LLM.
Returns "" when there are no loops so the digest is unchanged in the
common case.
Loops are rendered in the given order (detect_loops sorts them costliest
first). With *max_chars*, rendering stops at the first loop that would
take the section past that many chars, so the costliest ones are kept.
Returns "" when there are no loops, or none fits, so the digest is
unchanged in the common case.
"""
if not loops:
return ""
lines = [
header = [
"=== Detected Loops (HIGHEST PRIORITY) ===",
(
"These tool-call patterns REPEATED within a session — the most "
@@ -321,14 +330,21 @@ def format_loops_for_digest(loops: list[LoopPattern]) -> str:
),
"",
]
used = sum(len(ln) + 1 for ln in header) # each line plus its newline
entries: list[str] = []
for lp in loops:
lines.append(
f'- [{lp.kind}] {lp.tool}: "{lp.sample_input}" '
entry = (
f'- [{lp.kind}] {lp.tool}: "{lp.sample_input[:_SAMPLE_INPUT_CHARS]}" '
f"repeated {lp.count}x, ~{lp.wasted_tokens:,} tokens wasted "
f"(messages {lp.msg_indices})"
f"(messages {lp.msg_indices[:_DIGEST_MAX_INDICES]})"
)
lines.append("")
return "\n".join(lines)
used += len(entry) + 1
if max_chars is not None and used > max_chars:
break
entries.append(entry)
if not entries:
return ""
return "\n".join([*header, *entries, ""])
def _signature_tokens(signature: str) -> set[str]:
+13 -2
View File
@@ -475,11 +475,22 @@ class TestSessionAnalyzer:
assert result.recommendations == []
assert result.analysis_error == "API key not set"
@pytest.mark.parametrize(
"error",
[
"`claude -p` failed (exit 1):\nPrompt is too long",
# What codex exec printed on a real learn run: it refuses a prompt over 2**20 chars.
"`codex exec` failed (exit 1):\nError: turn/start: turn/start failed: Input exceeds "
"the maximum length of 1048576 characters. (code -32602), data: "
'{"input_error_code":"input_too_large","max_chars":1048576,"actual_chars":1786526}',
],
ids=["claude", "codex"],
)
@patch("headroom.learn.analyzer._call_llm")
def test_retries_smaller_digest_on_prompt_too_long(self, mock_call_llm: MagicMock):
def test_retries_smaller_digest_on_prompt_too_long(self, mock_call_llm: MagicMock, error):
"""A context-overflow error rebuilds the digest at half the budget and retries."""
mock_call_llm.side_effect = [
RuntimeError("`claude -p` failed (exit 1):\nPrompt is too long"),
RuntimeError(error),
{"context_file_rules": [], "memory_file_rules": []},
]
+42 -8
View File
@@ -151,22 +151,56 @@ class TestDigestSurfacesLoops:
digest = _build_digest(_project(), [one_off_error_session()])
assert "Detected Loops" not in digest
def test_loop_section_stays_inside_the_digest_budget(self):
@pytest.mark.parametrize("max_tokens", [80_000, 10_000])
@pytest.mark.parametrize(
("tool", "sample"),
[
("Bash", "x" * 120),
("Grep", "x" * 2000), # a long search pattern
("Glob", "src/" * 500 + "*.py"), # a long path
],
ids=["bash", "long-grep-pattern", "long-glob-path"],
)
def test_loop_section_stays_inside_the_digest_budget(self, tool, sample, max_tokens):
# Rendered whole, thousands of loops with hundreds of indices each made
# a 1.79M-char prompt, over Codex's 1,048,576-char input cap.
loops = [
LoopPattern("Bash", "s", "x" * 120, 300, False, 1_000_000 - i, list(range(300)))
LoopPattern(tool, "s", sample, 300, False, 1_000_000 - i, list(range(300)))
for i in range(2000)
]
full = _build_digest(_project(), [], loops=loops)
small = _build_digest(_project(), [], loops=loops, max_tokens=10_000)
assert len(full) < 80_000 # a quarter of the 80k-token budget's chars
assert len(small) < 10_000
digest = _build_digest(_project(), [], loops=loops, max_tokens=max_tokens)
section = digest[digest.index("=== Detected Loops") :]
assert len(section) <= max_tokens # a quarter of the budget's chars
# The costliest loops are the ones kept, and the caller's list is intact.
assert "~1,000,000 tokens wasted" in small
assert "~999,000 tokens wasted" not in full
assert "~1,000,000 tokens wasted" in section
assert "~999,000 tokens wasted" not in section
assert loops[0].sample_input == sample
assert len(loops[0].msg_indices) == 300
@pytest.mark.parametrize("max_tokens", [80_000, 10_000])
def test_loops_fit_beside_a_large_prior_patterns_section(self, tmp_path, max_tokens):
# The event stream cannot trim sections placed ahead of it, so the
# loops give way to the prior patterns, which are never cut.
bullets = "".join(f"- rule {i:06d} {'y' * 80}\n" for i in range(max_tokens * 4 // 100))
claude_md = tmp_path / "CLAUDE.md"
claude_md.write_text(
f"<!-- headroom:learn:start -->\n{bullets}<!-- headroom:learn:end -->\n"
)
project = ProjectInfo(
name="proj",
project_path=tmp_path,
data_path=tmp_path / "data",
context_file=claude_md,
)
loops = [
LoopPattern("Grep", "s", "x" * 2000, 300, False, 1_000_000 - i, list(range(300)))
for i in range(25)
]
digest = _build_digest(project, [], loops=loops, max_tokens=max_tokens)
assert bullets.splitlines()[-1] in digest
assert "~1,000,000 tokens wasted" in digest
assert len(digest) <= max_tokens * 4
# =============================================================================
# apply_loop_weighting