mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(learn): budget the digest's rendered loop section, retry Codex overflows
The loop cap counted loops (max_tokens // 400), assuming a ~100-token line. _build_digest also takes caller-supplied loops, and nothing bounded their rendered text: 25 Grep loops with 2,000-char patterns rendered 54,080 chars against the 40,000-char budget of a 10k-token retry. The event-stream check comes after the section, so it could not repair that. format_loops_for_digest now takes max_chars and stops at the first loop whose line would pass it, so the costliest loops are kept. Each line shows its sample input cut to 120 chars (the cut detect_loops already makes) and its first 20 message indices; the caller's LoopPatterns are not touched. _build_digest gives the section a quarter of the char budget, less what the header and the prior learned patterns leave short. The prior patterns are never cut: a re-emitted section replaces its prior one wholesale, so a partial baseline would drop bullets from the file. _is_prompt_too_long also matches Codex's "Input exceeds the maximum length" refusal, so a Codex overflow gets the smaller-digest retry. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+27
-30
@@ -23,7 +23,6 @@ import subprocess
|
||||
import threading
|
||||
import time
|
||||
import typing
|
||||
from dataclasses import replace
|
||||
|
||||
from headroom._subprocess import Popen, run
|
||||
|
||||
@@ -53,13 +52,6 @@ _MAX_DIGEST_TOKENS = 80_000 # Budget for the digest (leave room for prompt + ou
|
||||
# budget halves on each "prompt is too long" failure; below this the digest
|
||||
# carries too little signal to produce useful recommendations.
|
||||
_MIN_DIGEST_TOKENS = 10_000
|
||||
# The loop section precedes the budgeted event stream, so it is bounded on its
|
||||
# own: one loop per this many digest tokens (a rendered loop line is ~100
|
||||
# tokens, so the section stays under a quarter of the budget and halves with it
|
||||
# on a retry), each listing its first _MAX_LOOP_INDICES message indices beside
|
||||
# its repeat count.
|
||||
_DIGEST_TOKENS_PER_LOOP = 400
|
||||
_MAX_LOOP_INDICES = 20
|
||||
|
||||
# CLI tools to try when no API key is set (checked in order).
|
||||
# Each entry: (binary_name, model_identifier, command_prefix). The claude-cli
|
||||
@@ -344,8 +336,9 @@ def _is_prompt_too_long(exc: Exception) -> bool:
|
||||
|
||||
Matches the known phrasings across backends: claude CLI ("Prompt is too
|
||||
long"), Anthropic via LiteLLM ("prompt is too long: N tokens > M maximum"),
|
||||
OpenAI ("context_length_exceeded" / "maximum context length"), and Gemini
|
||||
("input token count exceeds the maximum").
|
||||
OpenAI ("context_length_exceeded" / "maximum context length"), Gemini
|
||||
("input token count exceeds the maximum"), and the codex CLI ("Input exceeds
|
||||
the maximum length of 1048576 characters").
|
||||
"""
|
||||
msg = str(exc).lower()
|
||||
return any(
|
||||
@@ -356,6 +349,7 @@ def _is_prompt_too_long(exc: Exception) -> bool:
|
||||
"maximum context length",
|
||||
"context window",
|
||||
"token count exceeds",
|
||||
"input exceeds the maximum length",
|
||||
)
|
||||
)
|
||||
|
||||
@@ -400,29 +394,32 @@ def _build_digest(
|
||||
lines.append(f"Tokens used: {total_tokens_in:,} in / {total_tokens_out:,} out")
|
||||
lines.append("")
|
||||
|
||||
# Detected loops first — the most expensive waste pattern, so the LLM sees
|
||||
# it before the (budget-truncatable) per-session event stream. Only the
|
||||
# costliest fit (detect_loops sorts by waste): rendered whole, every loop
|
||||
# with every index, the section alone outgrew the digest on a machine with
|
||||
# years of Codex sessions, which learn scans as one project.
|
||||
loop_section = format_loops_for_digest(
|
||||
[
|
||||
replace(lp, msg_indices=lp.msg_indices[:_MAX_LOOP_INDICES])
|
||||
for lp in loops[: max_tokens // _DIGEST_TOKENS_PER_LOOP]
|
||||
]
|
||||
)
|
||||
if loop_section:
|
||||
lines.append(loop_section)
|
||||
|
||||
# Prior learned patterns (if any) — gives the LLM the current baseline so
|
||||
# it can produce complete updated sections instead of condensed deltas.
|
||||
prior_section = _build_prior_patterns_section(project)
|
||||
if prior_section:
|
||||
lines.append(prior_section)
|
||||
|
||||
# Budget tracking — stop adding events when we approach the limit
|
||||
# Rough estimate: 4 chars per token
|
||||
char_budget = max_tokens * 4
|
||||
|
||||
# Prior learned patterns (if any) — gives the LLM the current baseline so
|
||||
# it can produce complete updated sections instead of condensed deltas.
|
||||
# Never cut: a re-emitted section replaces its prior one wholesale, so a
|
||||
# partial baseline would drop the missing bullets from the written file.
|
||||
prior_section = _build_prior_patterns_section(project)
|
||||
|
||||
# Detected loops first — the most expensive waste pattern, so the LLM sees
|
||||
# it before the (budget-truncatable) per-session event stream. The event
|
||||
# stream cannot trim a section placed ahead of it, so the loops get their
|
||||
# own cap: a quarter of the budget, less whatever the header and prior
|
||||
# patterns leave short. Rendered whole, every loop with every index, the
|
||||
# section alone outgrew the digest on a machine with years of Codex
|
||||
# sessions, which learn scans as one project.
|
||||
pre_event_chars = sum(len(ln) for ln in lines) + len(prior_section)
|
||||
loop_section = format_loops_for_digest(
|
||||
loops, max_chars=min(char_budget // 4, char_budget - pre_event_chars)
|
||||
)
|
||||
if loop_section:
|
||||
lines.append(loop_section)
|
||||
if prior_section:
|
||||
lines.append(prior_section)
|
||||
|
||||
chars_used = sum(len(ln) for ln in lines)
|
||||
|
||||
for session in sessions:
|
||||
|
||||
+26
-10
@@ -69,6 +69,11 @@ _PAGINATION_RE = re.compile("|".join(_PAGINATION_PATTERNS), re.IGNORECASE)
|
||||
# otherwise identical commands do not split a loop into singletons.
|
||||
_INT_RE = re.compile(r"\b\d+\b")
|
||||
|
||||
# A loop's digest line shows its sample input cut to this many chars and its
|
||||
# first _DIGEST_MAX_INDICES message indices beside its full repeat count.
|
||||
_SAMPLE_INPUT_CHARS = 120
|
||||
_DIGEST_MAX_INDICES = 20
|
||||
|
||||
|
||||
@dataclass
|
||||
class LoopPattern:
|
||||
@@ -290,7 +295,7 @@ def detect_loops(
|
||||
LoopPattern(
|
||||
tool=calls[0].name,
|
||||
signature=_fuzzy_signature(signatures[key]),
|
||||
sample_input=calls[0].input_summary[:120],
|
||||
sample_input=calls[0].input_summary[:_SAMPLE_INPUT_CHARS],
|
||||
count=count,
|
||||
is_error_loop=is_error_loop,
|
||||
wasted_tokens=wasted,
|
||||
@@ -302,15 +307,19 @@ def detect_loops(
|
||||
return loops
|
||||
|
||||
|
||||
def format_loops_for_digest(loops: list[LoopPattern]) -> str:
|
||||
def format_loops_for_digest(loops: list[LoopPattern], max_chars: int | None = None) -> str:
|
||||
"""Render detected loops as a high-priority digest section for the LLM.
|
||||
|
||||
Returns "" when there are no loops so the digest is unchanged in the
|
||||
common case.
|
||||
Loops are rendered in the given order (detect_loops sorts them costliest
|
||||
first). With *max_chars*, rendering stops at the first loop that would
|
||||
take the section past that many chars, so the costliest ones are kept.
|
||||
|
||||
Returns "" when there are no loops, or none fits, so the digest is
|
||||
unchanged in the common case.
|
||||
"""
|
||||
if not loops:
|
||||
return ""
|
||||
lines = [
|
||||
header = [
|
||||
"=== Detected Loops (HIGHEST PRIORITY) ===",
|
||||
(
|
||||
"These tool-call patterns REPEATED within a session — the most "
|
||||
@@ -321,14 +330,21 @@ def format_loops_for_digest(loops: list[LoopPattern]) -> str:
|
||||
),
|
||||
"",
|
||||
]
|
||||
used = sum(len(ln) + 1 for ln in header) # each line plus its newline
|
||||
entries: list[str] = []
|
||||
for lp in loops:
|
||||
lines.append(
|
||||
f'- [{lp.kind}] {lp.tool}: "{lp.sample_input}" '
|
||||
entry = (
|
||||
f'- [{lp.kind}] {lp.tool}: "{lp.sample_input[:_SAMPLE_INPUT_CHARS]}" '
|
||||
f"repeated {lp.count}x, ~{lp.wasted_tokens:,} tokens wasted "
|
||||
f"(messages {lp.msg_indices})"
|
||||
f"(messages {lp.msg_indices[:_DIGEST_MAX_INDICES]})"
|
||||
)
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
used += len(entry) + 1
|
||||
if max_chars is not None and used > max_chars:
|
||||
break
|
||||
entries.append(entry)
|
||||
if not entries:
|
||||
return ""
|
||||
return "\n".join([*header, *entries, ""])
|
||||
|
||||
|
||||
def _signature_tokens(signature: str) -> set[str]:
|
||||
|
||||
@@ -475,11 +475,22 @@ class TestSessionAnalyzer:
|
||||
assert result.recommendations == []
|
||||
assert result.analysis_error == "API key not set"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
"`claude -p` failed (exit 1):\nPrompt is too long",
|
||||
# What codex exec printed on a real learn run: it refuses a prompt over 2**20 chars.
|
||||
"`codex exec` failed (exit 1):\nError: turn/start: turn/start failed: Input exceeds "
|
||||
"the maximum length of 1048576 characters. (code -32602), data: "
|
||||
'{"input_error_code":"input_too_large","max_chars":1048576,"actual_chars":1786526}',
|
||||
],
|
||||
ids=["claude", "codex"],
|
||||
)
|
||||
@patch("headroom.learn.analyzer._call_llm")
|
||||
def test_retries_smaller_digest_on_prompt_too_long(self, mock_call_llm: MagicMock):
|
||||
def test_retries_smaller_digest_on_prompt_too_long(self, mock_call_llm: MagicMock, error):
|
||||
"""A context-overflow error rebuilds the digest at half the budget and retries."""
|
||||
mock_call_llm.side_effect = [
|
||||
RuntimeError("`claude -p` failed (exit 1):\nPrompt is too long"),
|
||||
RuntimeError(error),
|
||||
{"context_file_rules": [], "memory_file_rules": []},
|
||||
]
|
||||
|
||||
|
||||
@@ -151,22 +151,56 @@ class TestDigestSurfacesLoops:
|
||||
digest = _build_digest(_project(), [one_off_error_session()])
|
||||
assert "Detected Loops" not in digest
|
||||
|
||||
def test_loop_section_stays_inside_the_digest_budget(self):
|
||||
@pytest.mark.parametrize("max_tokens", [80_000, 10_000])
|
||||
@pytest.mark.parametrize(
|
||||
("tool", "sample"),
|
||||
[
|
||||
("Bash", "x" * 120),
|
||||
("Grep", "x" * 2000), # a long search pattern
|
||||
("Glob", "src/" * 500 + "*.py"), # a long path
|
||||
],
|
||||
ids=["bash", "long-grep-pattern", "long-glob-path"],
|
||||
)
|
||||
def test_loop_section_stays_inside_the_digest_budget(self, tool, sample, max_tokens):
|
||||
# Rendered whole, thousands of loops with hundreds of indices each made
|
||||
# a 1.79M-char prompt, over Codex's 1,048,576-char input cap.
|
||||
loops = [
|
||||
LoopPattern("Bash", "s", "x" * 120, 300, False, 1_000_000 - i, list(range(300)))
|
||||
LoopPattern(tool, "s", sample, 300, False, 1_000_000 - i, list(range(300)))
|
||||
for i in range(2000)
|
||||
]
|
||||
full = _build_digest(_project(), [], loops=loops)
|
||||
small = _build_digest(_project(), [], loops=loops, max_tokens=10_000)
|
||||
assert len(full) < 80_000 # a quarter of the 80k-token budget's chars
|
||||
assert len(small) < 10_000
|
||||
digest = _build_digest(_project(), [], loops=loops, max_tokens=max_tokens)
|
||||
section = digest[digest.index("=== Detected Loops") :]
|
||||
assert len(section) <= max_tokens # a quarter of the budget's chars
|
||||
# The costliest loops are the ones kept, and the caller's list is intact.
|
||||
assert "~1,000,000 tokens wasted" in small
|
||||
assert "~999,000 tokens wasted" not in full
|
||||
assert "~1,000,000 tokens wasted" in section
|
||||
assert "~999,000 tokens wasted" not in section
|
||||
assert loops[0].sample_input == sample
|
||||
assert len(loops[0].msg_indices) == 300
|
||||
|
||||
@pytest.mark.parametrize("max_tokens", [80_000, 10_000])
|
||||
def test_loops_fit_beside_a_large_prior_patterns_section(self, tmp_path, max_tokens):
|
||||
# The event stream cannot trim sections placed ahead of it, so the
|
||||
# loops give way to the prior patterns, which are never cut.
|
||||
bullets = "".join(f"- rule {i:06d} {'y' * 80}\n" for i in range(max_tokens * 4 // 100))
|
||||
claude_md = tmp_path / "CLAUDE.md"
|
||||
claude_md.write_text(
|
||||
f"<!-- headroom:learn:start -->\n{bullets}<!-- headroom:learn:end -->\n"
|
||||
)
|
||||
project = ProjectInfo(
|
||||
name="proj",
|
||||
project_path=tmp_path,
|
||||
data_path=tmp_path / "data",
|
||||
context_file=claude_md,
|
||||
)
|
||||
loops = [
|
||||
LoopPattern("Grep", "s", "x" * 2000, 300, False, 1_000_000 - i, list(range(300)))
|
||||
for i in range(25)
|
||||
]
|
||||
digest = _build_digest(project, [], loops=loops, max_tokens=max_tokens)
|
||||
assert bullets.splitlines()[-1] in digest
|
||||
assert "~1,000,000 tokens wasted" in digest
|
||||
assert len(digest) <= max_tokens * 4
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# apply_loop_weighting
|
||||
|
||||
Reference in New Issue
Block a user