fix(learn): only replace a verbosity pin this CLI provably set

Equality with the previously learned level does not prove an earlier
--apply set HEADROOM_VERBOSITY_LEVEL: an operator who exported exactly
that level had it overwritten by the next --apply.

Record ownership instead. When --apply hot-pins the level it writes
verbosity_pin.json with the proxy's port, pid (from /health) and the level.
A later --apply replaces a pin only while that same proxy process still
holds that exact value. Hot overrides die with the process, so a restarted
proxy's pin, or a value changed since, stays the operator's. No pid, no
record.

Also only print the re-cache note when the pinned level actually changes.
This commit is contained in:
Kenneth Wong
2026-10-08 18:16:58 +08:00
parent c75145dd64
commit e10141c348
3 changed files with 157 additions and 29 deletions
+55 -21
View File
@@ -441,13 +441,13 @@ def _steered_level(raw: object) -> int | None:
return max(0, min(4, level))
def _query_proxy_verbosity(port: int | None = None) -> tuple[str | None, int | None]:
"""Best-effort: read a running local proxy's mode and pinned verbosity level.
def _query_proxy_verbosity(port: int | None = None) -> tuple[str | None, int | None, int | None]:
"""Best-effort: read a running local proxy's mode, pinned level and pid.
Both come from the ``/health`` config block, which the proxy includes for
loopback callers. Returns ``(mode, level)``: ``mode`` is ``None`` when the
proxy is unreachable or does not report it, and ``level`` is the level its
``HEADROOM_VERBOSITY_LEVEL`` pins, or ``None`` when that is unset.
All three come from the ``/health`` config block, which the proxy includes
for loopback callers. Returns ``(mode, level, pid)``, each ``None`` when the
proxy is unreachable or does not report it; ``level`` is the level its
``HEADROOM_VERBOSITY_LEVEL`` pins, ``None`` when that is unset.
"""
import json as _json
import urllib.error
@@ -458,14 +458,49 @@ def _query_proxy_verbosity(port: int | None = None) -> tuple[str | None, int | N
with urllib.request.urlopen(url, timeout=2) as response:
payload = _json.loads(response.read() or b"{}")
except (urllib.error.URLError, OSError, ValueError):
return None, None
return None, None, None
config = payload.get("config") if isinstance(payload, dict) else None
if not isinstance(config, dict):
return None, None
return None, None, None
mode = config.get("mode")
runtime = config.get("runtime_env")
level = runtime.get("HEADROOM_VERBOSITY_LEVEL") if isinstance(runtime, dict) else None
return mode if isinstance(mode, str) else None, _steered_level(level)
pid = config.get("pid")
return (
mode if isinstance(mode, str) else None,
_steered_level(level),
pid if isinstance(pid, int) else None,
)
#: Records the HEADROOM_VERBOSITY_LEVEL that ``--apply`` hot-pinned, and on
#: which proxy process. Hot overrides die with the process, so a pin is ours to
#: replace only while the same process (port + pid) still holds that value.
_VERBOSITY_PIN_FILE = "verbosity_pin.json"
def _pin_is_ours(ws: Path, port: int, pid: int | None, level: int | None) -> bool:
import json as _json
if pid is None or level is None:
return False
try:
record = _json.loads((ws / _VERBOSITY_PIN_FILE).read_text(encoding="utf-8"))
except (OSError, ValueError):
return False
return isinstance(record, dict) and record == {"port": port, "pid": pid, "level": level}
def _record_pin(ws: Path, port: int, pid: int | None, level: int) -> None:
import json as _json
path = ws / _VERBOSITY_PIN_FILE
if pid is None:
# Without a pid the pin cannot be attributed later; leave no record.
path.unlink(missing_ok=True)
return
record = {"port": port, "pid": pid, "level": level}
path.write_text(_json.dumps(record), encoding="utf-8", newline="\n")
def _activate_output_shaper(
@@ -632,9 +667,6 @@ def _run_verbosity(
ws = ensure_workspace_dir()
from datetime import datetime, timezone
from ..learn.verbosity import VerbosityProfile
previous = VerbosityProfile.load(ws / "verbosity.json")
best_profile.learned_at = datetime.now(timezone.utc).isoformat()
best_profile.save(ws / "verbosity.json")
# Seed the savings baseline: replace baseline, preserve any live
@@ -652,19 +684,21 @@ def _run_verbosity(
# Make --apply actually take effect: hot-enable a running proxy, and
# otherwise tell the user exactly how to turn it on.
level = best_profile.level
mode, pinned = _query_proxy_verbosity()
port = _local_proxy_port()
mode, pinned, pid = _query_proxy_verbosity(port)
# Cache mode never reads verbosity.json: a level that can change while
# conversations are open would bust their prefix cache, so the proxy
# steers at its startup level unless HEADROOM_VERBOSITY_LEVEL pins one
# (resolve_verbosity_level). Pin the learned level there; otherwise the
# proxy would keep steering at its default while we report this level.
# A pin equal to the previously learned level is taken to be the one an
# earlier --apply set (the proxy cannot say where a value came from), so
# it is replaced rather than left to shadow the new level. Any other pin
# is the operator's and is left alone.
earlier_apply = pinned is not None and previous is not None and pinned == previous.level
pin = str(level) if mode == "cache" and (pinned is None or earlier_apply) else None
status, shaper_port = _activate_output_shaper(verbosity_level=pin)
# An existing pin is the operator's and is left alone, unless the record
# from an earlier --apply shows this same proxy process holds the value
# that run pinned.
ours = _pin_is_ours(ws, port, pid, pinned)
pin = str(level) if mode == "cache" and (pinned is None or ours) else None
status, shaper_port = _activate_output_shaper(port, verbosity_level=pin)
if pin is not None and status == "live":
_record_pin(ws, shaper_port, pid, level)
cache_mode_hint = (
f"in cache mode (the default) also set HEADROOM_VERBOSITY_LEVEL={level}, "
"because cache mode does not read the learned level"
@@ -698,7 +732,7 @@ def _run_verbosity(
f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); "
f"level {level} is live now{how}."
)
if pin:
if pin and pinned != level:
click.echo(
" Conversations already open re-cache their prompt once, because "
"the steering text changed."
+97 -4
View File
@@ -676,6 +676,10 @@ def _no_local_proxy(*args, **kwargs): # noqa: ANN002, ANN003, ANN201
raise urllib.error.URLError("no proxy in tests")
# Where the stubbed local proxy listens; HEADROOM_PORT points the CLI at it.
_PROXY_PORT = 18787
class _FakeProxy:
"""A local proxy as ``urllib.request.urlopen`` sees it.
@@ -709,17 +713,22 @@ def _apply_learned_level(
proxy: _FakeProxy,
*,
level: int = 3,
pin_record: dict | None = None,
previous_level: int | None = None,
) -> str:
"""Run ``learn --verbosity --apply`` for one project that learns ``level``.
``previous_level`` seeds verbosity.json as an earlier ``--apply`` left it.
``pin_record`` seeds the record an earlier ``--apply`` leaves after pinning,
``previous_level`` the verbosity.json it saved. The proxy listens on
``_PROXY_PORT``.
"""
from headroom.proxy.output_savings import BaselineModel
workspace = tmp_path / "ws"
workspace.mkdir()
if pin_record is not None:
(workspace / "verbosity_pin.json").write_text(json.dumps(pin_record), newline="\n")
if previous_level is not None:
workspace.mkdir()
(workspace / "verbosity.json").write_text(
json.dumps({"verbosity_level": previous_level}), newline="\n"
)
@@ -750,6 +759,7 @@ def _apply_learned_level(
lambda session_paths, project_path, llm_judge=None: (profile, baseline),
)
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(workspace))
monkeypatch.setenv("HEADROOM_PORT", str(_PROXY_PORT))
monkeypatch.setattr("urllib.request.urlopen", proxy.urlopen)
result = runner.invoke(
@@ -772,6 +782,7 @@ def test_verbosity_apply_pins_the_level_on_a_cache_mode_proxy(
assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]
assert "level 3 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output
assert "HEADROOM_OUTPUT_SHAPER=1 HEADROOM_VERBOSITY_LEVEL=3 before" in output
assert "re-cache their prompt once" in output
def test_verbosity_apply_leaves_a_token_mode_proxy_reading_the_profile(
@@ -799,16 +810,98 @@ def test_verbosity_apply_does_not_override_an_explicit_level(
assert "it stays at level 2: HEADROOM_VERBOSITY_LEVEL is set there" in output
def test_verbosity_apply_records_the_pin_it_sets(
monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
) -> None:
proxy = _FakeProxy({"mode": "cache", "runtime_env": {}, "pid": 4242})
_apply_learned_level(monkeypatch, runner, tmp_path, proxy)
record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())
assert record == {"port": _PROXY_PORT, "pid": 4242, "level": 3}
def test_verbosity_apply_replaces_the_pin_an_earlier_apply_set(
monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
) -> None:
"""A re-learned level must not be shadowed by the previous run's own pin."""
proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}})
proxy = _FakeProxy(
{"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}
)
output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy, level=1, previous_level=3)
output = _apply_learned_level(
monkeypatch,
runner,
tmp_path,
proxy,
level=1,
pin_record={"port": _PROXY_PORT, "pid": 4242, "level": 3},
)
assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "1"}]
assert "level 1 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output
record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())
assert record == {"port": _PROXY_PORT, "pid": 4242, "level": 1}
def test_verbosity_apply_re_pinning_the_same_level_does_not_warn_about_re_caching(
monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
) -> None:
proxy = _FakeProxy(
{"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}
)
output = _apply_learned_level(
monkeypatch,
runner,
tmp_path,
proxy,
pin_record={"port": _PROXY_PORT, "pid": 4242, "level": 3},
)
assert "level 3 is live now" in output
assert "re-cache" not in output
@pytest.mark.parametrize(
"pin_record",
[
None, # the operator pinned the level that was learned before
{"port": _PROXY_PORT, "pid": 1111, "level": 3}, # pinned on a proxy since restarted
{"port": _PROXY_PORT, "pid": 4242, "level": 2}, # the operator re-pinned after us
],
ids=["no-record", "restarted-proxy", "changed-since"],
)
def test_verbosity_apply_keeps_a_pin_it_cannot_prove_it_set(
monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path, pin_record: dict | None
) -> None:
proxy = _FakeProxy(
{"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}
)
output = _apply_learned_level(
monkeypatch, runner, tmp_path, proxy, level=1, pin_record=pin_record, previous_level=3
)
assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]
assert "it stays at level 3: HEADROOM_VERBOSITY_LEVEL is set there" in output
@pytest.mark.windows_newline
def test_verbosity_pin_record_is_written_with_lf(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001
from headroom.cli import learn as learn_cli
calls: list[dict] = []
real_write_text = Path.write_text
def write_text(self, data, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202
calls.append(kwargs)
return real_write_text(self, data, *args, **kwargs)
monkeypatch.setattr(Path, "write_text", write_text)
learn_cli._record_pin(tmp_path, _PROXY_PORT, 4242, 3)
assert [call.get("newline") for call in calls] == ["\n"]
@pytest.mark.parametrize("raw", ["9", " 04 "])
+5 -4
View File
@@ -177,10 +177,11 @@ turns it on; it is available on every rollout channel). Running `--verbosity --a
A proxy in cache mode (the default) does not read `verbosity.json`: a level that
changes mid-conversation would bust the prompt cache, so it steers at its startup
level. On a cache-mode proxy `--apply` therefore also pins
`HEADROOM_VERBOSITY_LEVEL` to the learned level. A `HEADROOM_VERBOSITY_LEVEL`
that is already set is left alone, and `--apply` says so, unless it equals the
previously learned level. That one is taken to be an earlier `--apply`'s pin and
is replaced.
`HEADROOM_VERBOSITY_LEVEL` to the learned level, and records the pin (port,
proxy pid, level) in `verbosity_pin.json`. A `HEADROOM_VERBOSITY_LEVEL` that is
already set is left alone, and `--apply` says so. The one exception is a pin that
record proves an earlier `--apply` set: the same proxy process still holding the
same value. That pin is replaced.
To keep the shaper on across proxy restarts, export `HEADROOM_OUTPUT_SHAPER=1` before starting the proxy, plus `HEADROOM_VERBOSITY_LEVEL=<level>` in cache mode.