mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(learn): only replace a verbosity pin this CLI provably set
Equality with the previously learned level does not prove an earlier --apply set HEADROOM_VERBOSITY_LEVEL: an operator who exported exactly that level had it overwritten by the next --apply. Record ownership instead. When --apply hot-pins the level it writes verbosity_pin.json with the proxy's port, pid (from /health) and the level. A later --apply replaces a pin only while that same proxy process still holds that exact value. Hot overrides die with the process, so a restarted proxy's pin, or a value changed since, stays the operator's. No pid, no record. Also only print the re-cache note when the pinned level actually changes.
This commit is contained in:
+55
-21
@@ -441,13 +441,13 @@ def _steered_level(raw: object) -> int | None:
|
||||
return max(0, min(4, level))
|
||||
|
||||
|
||||
def _query_proxy_verbosity(port: int | None = None) -> tuple[str | None, int | None]:
|
||||
"""Best-effort: read a running local proxy's mode and pinned verbosity level.
|
||||
def _query_proxy_verbosity(port: int | None = None) -> tuple[str | None, int | None, int | None]:
|
||||
"""Best-effort: read a running local proxy's mode, pinned level and pid.
|
||||
|
||||
Both come from the ``/health`` config block, which the proxy includes for
|
||||
loopback callers. Returns ``(mode, level)``: ``mode`` is ``None`` when the
|
||||
proxy is unreachable or does not report it, and ``level`` is the level its
|
||||
``HEADROOM_VERBOSITY_LEVEL`` pins, or ``None`` when that is unset.
|
||||
All three come from the ``/health`` config block, which the proxy includes
|
||||
for loopback callers. Returns ``(mode, level, pid)``, each ``None`` when the
|
||||
proxy is unreachable or does not report it; ``level`` is the level its
|
||||
``HEADROOM_VERBOSITY_LEVEL`` pins, ``None`` when that is unset.
|
||||
"""
|
||||
import json as _json
|
||||
import urllib.error
|
||||
@@ -458,14 +458,49 @@ def _query_proxy_verbosity(port: int | None = None) -> tuple[str | None, int | N
|
||||
with urllib.request.urlopen(url, timeout=2) as response:
|
||||
payload = _json.loads(response.read() or b"{}")
|
||||
except (urllib.error.URLError, OSError, ValueError):
|
||||
return None, None
|
||||
return None, None, None
|
||||
config = payload.get("config") if isinstance(payload, dict) else None
|
||||
if not isinstance(config, dict):
|
||||
return None, None
|
||||
return None, None, None
|
||||
mode = config.get("mode")
|
||||
runtime = config.get("runtime_env")
|
||||
level = runtime.get("HEADROOM_VERBOSITY_LEVEL") if isinstance(runtime, dict) else None
|
||||
return mode if isinstance(mode, str) else None, _steered_level(level)
|
||||
pid = config.get("pid")
|
||||
return (
|
||||
mode if isinstance(mode, str) else None,
|
||||
_steered_level(level),
|
||||
pid if isinstance(pid, int) else None,
|
||||
)
|
||||
|
||||
|
||||
#: Records the HEADROOM_VERBOSITY_LEVEL that ``--apply`` hot-pinned, and on
|
||||
#: which proxy process. Hot overrides die with the process, so a pin is ours to
|
||||
#: replace only while the same process (port + pid) still holds that value.
|
||||
_VERBOSITY_PIN_FILE = "verbosity_pin.json"
|
||||
|
||||
|
||||
def _pin_is_ours(ws: Path, port: int, pid: int | None, level: int | None) -> bool:
|
||||
import json as _json
|
||||
|
||||
if pid is None or level is None:
|
||||
return False
|
||||
try:
|
||||
record = _json.loads((ws / _VERBOSITY_PIN_FILE).read_text(encoding="utf-8"))
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
return isinstance(record, dict) and record == {"port": port, "pid": pid, "level": level}
|
||||
|
||||
|
||||
def _record_pin(ws: Path, port: int, pid: int | None, level: int) -> None:
|
||||
import json as _json
|
||||
|
||||
path = ws / _VERBOSITY_PIN_FILE
|
||||
if pid is None:
|
||||
# Without a pid the pin cannot be attributed later; leave no record.
|
||||
path.unlink(missing_ok=True)
|
||||
return
|
||||
record = {"port": port, "pid": pid, "level": level}
|
||||
path.write_text(_json.dumps(record), encoding="utf-8", newline="\n")
|
||||
|
||||
|
||||
def _activate_output_shaper(
|
||||
@@ -632,9 +667,6 @@ def _run_verbosity(
|
||||
ws = ensure_workspace_dir()
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from ..learn.verbosity import VerbosityProfile
|
||||
|
||||
previous = VerbosityProfile.load(ws / "verbosity.json")
|
||||
best_profile.learned_at = datetime.now(timezone.utc).isoformat()
|
||||
best_profile.save(ws / "verbosity.json")
|
||||
# Seed the savings baseline: replace baseline, preserve any live
|
||||
@@ -652,19 +684,21 @@ def _run_verbosity(
|
||||
# Make --apply actually take effect: hot-enable a running proxy, and
|
||||
# otherwise tell the user exactly how to turn it on.
|
||||
level = best_profile.level
|
||||
mode, pinned = _query_proxy_verbosity()
|
||||
port = _local_proxy_port()
|
||||
mode, pinned, pid = _query_proxy_verbosity(port)
|
||||
# Cache mode never reads verbosity.json: a level that can change while
|
||||
# conversations are open would bust their prefix cache, so the proxy
|
||||
# steers at its startup level unless HEADROOM_VERBOSITY_LEVEL pins one
|
||||
# (resolve_verbosity_level). Pin the learned level there; otherwise the
|
||||
# proxy would keep steering at its default while we report this level.
|
||||
# A pin equal to the previously learned level is taken to be the one an
|
||||
# earlier --apply set (the proxy cannot say where a value came from), so
|
||||
# it is replaced rather than left to shadow the new level. Any other pin
|
||||
# is the operator's and is left alone.
|
||||
earlier_apply = pinned is not None and previous is not None and pinned == previous.level
|
||||
pin = str(level) if mode == "cache" and (pinned is None or earlier_apply) else None
|
||||
status, shaper_port = _activate_output_shaper(verbosity_level=pin)
|
||||
# An existing pin is the operator's and is left alone, unless the record
|
||||
# from an earlier --apply shows this same proxy process holds the value
|
||||
# that run pinned.
|
||||
ours = _pin_is_ours(ws, port, pid, pinned)
|
||||
pin = str(level) if mode == "cache" and (pinned is None or ours) else None
|
||||
status, shaper_port = _activate_output_shaper(port, verbosity_level=pin)
|
||||
if pin is not None and status == "live":
|
||||
_record_pin(ws, shaper_port, pid, level)
|
||||
cache_mode_hint = (
|
||||
f"in cache mode (the default) also set HEADROOM_VERBOSITY_LEVEL={level}, "
|
||||
"because cache mode does not read the learned level"
|
||||
@@ -698,7 +732,7 @@ def _run_verbosity(
|
||||
f"\n ✓ Output shaper enabled on the running proxy (port {shaper_port}); "
|
||||
f"level {level} is live now{how}."
|
||||
)
|
||||
if pin:
|
||||
if pin and pinned != level:
|
||||
click.echo(
|
||||
" Conversations already open re-cache their prompt once, because "
|
||||
"the steering text changed."
|
||||
|
||||
+97
-4
@@ -676,6 +676,10 @@ def _no_local_proxy(*args, **kwargs): # noqa: ANN002, ANN003, ANN201
|
||||
raise urllib.error.URLError("no proxy in tests")
|
||||
|
||||
|
||||
# Where the stubbed local proxy listens; HEADROOM_PORT points the CLI at it.
|
||||
_PROXY_PORT = 18787
|
||||
|
||||
|
||||
class _FakeProxy:
|
||||
"""A local proxy as ``urllib.request.urlopen`` sees it.
|
||||
|
||||
@@ -709,17 +713,22 @@ def _apply_learned_level(
|
||||
proxy: _FakeProxy,
|
||||
*,
|
||||
level: int = 3,
|
||||
pin_record: dict | None = None,
|
||||
previous_level: int | None = None,
|
||||
) -> str:
|
||||
"""Run ``learn --verbosity --apply`` for one project that learns ``level``.
|
||||
|
||||
``previous_level`` seeds verbosity.json as an earlier ``--apply`` left it.
|
||||
``pin_record`` seeds the record an earlier ``--apply`` leaves after pinning,
|
||||
``previous_level`` the verbosity.json it saved. The proxy listens on
|
||||
``_PROXY_PORT``.
|
||||
"""
|
||||
from headroom.proxy.output_savings import BaselineModel
|
||||
|
||||
workspace = tmp_path / "ws"
|
||||
workspace.mkdir()
|
||||
if pin_record is not None:
|
||||
(workspace / "verbosity_pin.json").write_text(json.dumps(pin_record), newline="\n")
|
||||
if previous_level is not None:
|
||||
workspace.mkdir()
|
||||
(workspace / "verbosity.json").write_text(
|
||||
json.dumps({"verbosity_level": previous_level}), newline="\n"
|
||||
)
|
||||
@@ -750,6 +759,7 @@ def _apply_learned_level(
|
||||
lambda session_paths, project_path, llm_judge=None: (profile, baseline),
|
||||
)
|
||||
monkeypatch.setenv("HEADROOM_WORKSPACE_DIR", str(workspace))
|
||||
monkeypatch.setenv("HEADROOM_PORT", str(_PROXY_PORT))
|
||||
monkeypatch.setattr("urllib.request.urlopen", proxy.urlopen)
|
||||
|
||||
result = runner.invoke(
|
||||
@@ -772,6 +782,7 @@ def test_verbosity_apply_pins_the_level_on_a_cache_mode_proxy(
|
||||
assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "3"}]
|
||||
assert "level 3 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output
|
||||
assert "HEADROOM_OUTPUT_SHAPER=1 HEADROOM_VERBOSITY_LEVEL=3 before" in output
|
||||
assert "re-cache their prompt once" in output
|
||||
|
||||
|
||||
def test_verbosity_apply_leaves_a_token_mode_proxy_reading_the_profile(
|
||||
@@ -799,16 +810,98 @@ def test_verbosity_apply_does_not_override_an_explicit_level(
|
||||
assert "it stays at level 2: HEADROOM_VERBOSITY_LEVEL is set there" in output
|
||||
|
||||
|
||||
def test_verbosity_apply_records_the_pin_it_sets(
|
||||
monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
|
||||
) -> None:
|
||||
proxy = _FakeProxy({"mode": "cache", "runtime_env": {}, "pid": 4242})
|
||||
|
||||
_apply_learned_level(monkeypatch, runner, tmp_path, proxy)
|
||||
|
||||
record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())
|
||||
assert record == {"port": _PROXY_PORT, "pid": 4242, "level": 3}
|
||||
|
||||
|
||||
def test_verbosity_apply_replaces_the_pin_an_earlier_apply_set(
|
||||
monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
|
||||
) -> None:
|
||||
"""A re-learned level must not be shadowed by the previous run's own pin."""
|
||||
proxy = _FakeProxy({"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}})
|
||||
proxy = _FakeProxy(
|
||||
{"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}
|
||||
)
|
||||
|
||||
output = _apply_learned_level(monkeypatch, runner, tmp_path, proxy, level=1, previous_level=3)
|
||||
output = _apply_learned_level(
|
||||
monkeypatch,
|
||||
runner,
|
||||
tmp_path,
|
||||
proxy,
|
||||
level=1,
|
||||
pin_record={"port": _PROXY_PORT, "pid": 4242, "level": 3},
|
||||
)
|
||||
|
||||
assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1", "HEADROOM_VERBOSITY_LEVEL": "1"}]
|
||||
assert "level 1 is live now (pinned with HEADROOM_VERBOSITY_LEVEL" in output
|
||||
record = json.loads((tmp_path / "ws" / "verbosity_pin.json").read_text())
|
||||
assert record == {"port": _PROXY_PORT, "pid": 4242, "level": 1}
|
||||
|
||||
|
||||
def test_verbosity_apply_re_pinning_the_same_level_does_not_warn_about_re_caching(
|
||||
monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path
|
||||
) -> None:
|
||||
proxy = _FakeProxy(
|
||||
{"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}
|
||||
)
|
||||
|
||||
output = _apply_learned_level(
|
||||
monkeypatch,
|
||||
runner,
|
||||
tmp_path,
|
||||
proxy,
|
||||
pin_record={"port": _PROXY_PORT, "pid": 4242, "level": 3},
|
||||
)
|
||||
|
||||
assert "level 3 is live now" in output
|
||||
assert "re-cache" not in output
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pin_record",
|
||||
[
|
||||
None, # the operator pinned the level that was learned before
|
||||
{"port": _PROXY_PORT, "pid": 1111, "level": 3}, # pinned on a proxy since restarted
|
||||
{"port": _PROXY_PORT, "pid": 4242, "level": 2}, # the operator re-pinned after us
|
||||
],
|
||||
ids=["no-record", "restarted-proxy", "changed-since"],
|
||||
)
|
||||
def test_verbosity_apply_keeps_a_pin_it_cannot_prove_it_set(
|
||||
monkeypatch: pytest.MonkeyPatch, runner: CliRunner, tmp_path: Path, pin_record: dict | None
|
||||
) -> None:
|
||||
proxy = _FakeProxy(
|
||||
{"mode": "cache", "runtime_env": {"HEADROOM_VERBOSITY_LEVEL": "3"}, "pid": 4242}
|
||||
)
|
||||
|
||||
output = _apply_learned_level(
|
||||
monkeypatch, runner, tmp_path, proxy, level=1, pin_record=pin_record, previous_level=3
|
||||
)
|
||||
|
||||
assert proxy.posted == [{"HEADROOM_OUTPUT_SHAPER": "1"}]
|
||||
assert "it stays at level 3: HEADROOM_VERBOSITY_LEVEL is set there" in output
|
||||
|
||||
|
||||
@pytest.mark.windows_newline
|
||||
def test_verbosity_pin_record_is_written_with_lf(tmp_path: Path, monkeypatch) -> None: # noqa: ANN001
|
||||
from headroom.cli import learn as learn_cli
|
||||
|
||||
calls: list[dict] = []
|
||||
real_write_text = Path.write_text
|
||||
|
||||
def write_text(self, data, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202
|
||||
calls.append(kwargs)
|
||||
return real_write_text(self, data, *args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(Path, "write_text", write_text)
|
||||
learn_cli._record_pin(tmp_path, _PROXY_PORT, 4242, 3)
|
||||
|
||||
assert [call.get("newline") for call in calls] == ["\n"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raw", ["9", " 04 "])
|
||||
|
||||
+5
-4
@@ -177,10 +177,11 @@ turns it on; it is available on every rollout channel). Running `--verbosity --a
|
||||
A proxy in cache mode (the default) does not read `verbosity.json`: a level that
|
||||
changes mid-conversation would bust the prompt cache, so it steers at its startup
|
||||
level. On a cache-mode proxy `--apply` therefore also pins
|
||||
`HEADROOM_VERBOSITY_LEVEL` to the learned level. A `HEADROOM_VERBOSITY_LEVEL`
|
||||
that is already set is left alone, and `--apply` says so, unless it equals the
|
||||
previously learned level. That one is taken to be an earlier `--apply`'s pin and
|
||||
is replaced.
|
||||
`HEADROOM_VERBOSITY_LEVEL` to the learned level, and records the pin (port,
|
||||
proxy pid, level) in `verbosity_pin.json`. A `HEADROOM_VERBOSITY_LEVEL` that is
|
||||
already set is left alone, and `--apply` says so. The one exception is a pin that
|
||||
record proves an earlier `--apply` set: the same proxy process still holding the
|
||||
same value. That pin is replaced.
|
||||
|
||||
To keep the shaper on across proxy restarts, export `HEADROOM_OUTPUT_SHAPER=1` before starting the proxy, plus `HEADROOM_VERBOSITY_LEVEL=<level>` in cache mode.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user