mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(claude): serialize stale recovery with settings writers
This commit is contained in:
+39
-28
@@ -1400,9 +1400,9 @@ def _locked_file(lock_file: Any) -> Any:
|
||||
import msvcrt
|
||||
|
||||
# msvcrt.locking operates on bytes from the current file position.
|
||||
lock_file.seek(0)
|
||||
if lock_file.read(1) == b"":
|
||||
lock_file.seek(0)
|
||||
# Reading the locked byte fails on Windows before a competing
|
||||
# writer can enter the retry loop. Check metadata instead.
|
||||
if os.fstat(lock_file.fileno()).st_size == 0:
|
||||
lock_file.write(b"0")
|
||||
lock_file.flush()
|
||||
lock_file.seek(0)
|
||||
@@ -1728,33 +1728,41 @@ def _check_and_clear_stale_wrap_marker(settings_path: Path, *, key: str) -> str
|
||||
Called before writing a fresh base_url entry so a crashed wrap session's
|
||||
leftover doesn't get treated as this session's own state to restore later.
|
||||
"""
|
||||
# Preserve the default no-marker path without creating lock artifacts.
|
||||
marker = _read_wrap_marker(settings_path)
|
||||
if marker is None or marker.get("key") != key or not _wrap_marker_is_stale(marker):
|
||||
return None
|
||||
port = marker.get("port")
|
||||
if not isinstance(port, int) or isinstance(port, bool):
|
||||
return None
|
||||
expected_url = f"http://127.0.0.1:{port}"
|
||||
if key == "ANTHROPIC_FOUNDRY_BASE_URL":
|
||||
expected_url = _foundry_proxy_url(expected_url)
|
||||
try:
|
||||
settings = json.loads(_read_text(settings_path))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
return None
|
||||
env_settings = settings.get("env") if isinstance(settings, dict) else None
|
||||
current_url = env_settings.get(key) if isinstance(env_settings, dict) else None
|
||||
if current_url != expected_url:
|
||||
# The user (or a newer session) has replaced the crashed writer's URL.
|
||||
# Retire its stale marker without changing the current settings.
|
||||
_clear_wrap_marker(settings_path, key=key)
|
||||
return None
|
||||
previous = marker.get("previous")
|
||||
click.echo(
|
||||
f"headroom: clearing stale {key} left by crashed wrap session (pid {marker.get('pid')})",
|
||||
err=True,
|
||||
)
|
||||
_restore_claude_wrap_base_url(previous, settings_path=settings_path, _key_override=key)
|
||||
return previous
|
||||
with _wrap_settings_lock(settings_path):
|
||||
marker = _read_wrap_marker(settings_path)
|
||||
if marker is None or marker.get("key") != key or not _wrap_marker_is_stale(marker):
|
||||
return None
|
||||
port = marker.get("port")
|
||||
if not isinstance(port, int) or isinstance(port, bool):
|
||||
return None
|
||||
expected_url = f"http://127.0.0.1:{port}"
|
||||
if key == "ANTHROPIC_FOUNDRY_BASE_URL":
|
||||
expected_url = _foundry_proxy_url(expected_url)
|
||||
try:
|
||||
settings = json.loads(_read_text(settings_path))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
return None
|
||||
env_settings = settings.get("env") if isinstance(settings, dict) else None
|
||||
current_url = env_settings.get(key) if isinstance(env_settings, dict) else None
|
||||
if current_url != expected_url:
|
||||
# The user (or a newer session) has replaced the crashed writer's URL.
|
||||
# Retire its stale marker without changing the current settings.
|
||||
if _read_wrap_marker(settings_path) == marker:
|
||||
_clear_wrap_marker(settings_path, key=key)
|
||||
return None
|
||||
previous = marker.get("previous")
|
||||
click.echo(
|
||||
f"headroom: clearing stale {key} left by crashed wrap session (pid {marker.get('pid')})",
|
||||
err=True,
|
||||
)
|
||||
_restore_claude_wrap_base_url(
|
||||
previous, settings_path=settings_path, _key_override=key, _lock_held=True
|
||||
)
|
||||
return previous
|
||||
|
||||
|
||||
def _check_and_clear_dead_wrap_marker(settings_path: Path, *, key: str) -> str | None:
|
||||
@@ -2058,6 +2066,7 @@ def _restore_claude_wrap_base_url(
|
||||
_key_override: str | None = None,
|
||||
force: bool = False,
|
||||
dead_ports: frozenset[int] = frozenset(),
|
||||
_lock_held: bool = False,
|
||||
) -> None:
|
||||
"""Restore (or remove) the env key written by _write_claude_wrap_base_url.
|
||||
|
||||
@@ -2073,11 +2082,13 @@ def _restore_claude_wrap_base_url(
|
||||
(``unwrap``), and ``dead_ports`` to name proxy ports already proven dead so
|
||||
holders that outlived their proxy stop counting as live.
|
||||
"""
|
||||
from contextlib import nullcontext
|
||||
|
||||
path = settings_path or (Path.cwd() / ".claude" / "settings.local.json")
|
||||
key = _key_override or _claude_wrap_base_url_env_key(
|
||||
foundry_mode=foundry_mode, vertex_mode=vertex_mode
|
||||
)
|
||||
with _wrap_settings_lock(path):
|
||||
with nullcontext() if _lock_held else _wrap_settings_lock(path):
|
||||
# Another live wrap session in this project may still be using the key.
|
||||
# Restoring underneath it silently unroutes a running session -- traffic
|
||||
# bypasses the proxy with no error anywhere (#3205).
|
||||
|
||||
@@ -555,3 +555,66 @@ def test_stale_marker_preserves_manually_repaired_url(tmp_path, foundry_mode, ve
|
||||
wrap_cli._check_and_clear_stale_wrap_marker(path, key=key)
|
||||
assert path.read_bytes() == before
|
||||
assert not marker_path.exists()
|
||||
|
||||
|
||||
def test_stale_recovery_serializes_with_new_wrap_writer(tmp_path, monkeypatch):
|
||||
import threading
|
||||
|
||||
path = _settings(tmp_path)
|
||||
path.parent.mkdir(parents=True)
|
||||
wrap_cli._write_claude_wrap_base_url("http://127.0.0.1:9200", settings_path=path, port=9200)
|
||||
marker_path = wrap_cli._wrap_marker_path(path)
|
||||
marker = json.loads(marker_path.read_text(encoding="utf-8"))
|
||||
marker["pid"] = 999_999_999
|
||||
marker_path.write_text(json.dumps(marker), encoding="utf-8")
|
||||
path.write_text('{"env":{"ANTHROPIC_BASE_URL":"https://repaired.example"}}', encoding="utf-8")
|
||||
compared = threading.Event()
|
||||
resume = threading.Event()
|
||||
writer_done = threading.Event()
|
||||
errors = []
|
||||
read_text = wrap_cli._read_text
|
||||
|
||||
def paused_read(target):
|
||||
content = read_text(target)
|
||||
if threading.current_thread().name == "stale-recovery" and target == path:
|
||||
compared.set()
|
||||
assert resume.wait(5)
|
||||
return content
|
||||
|
||||
def recover():
|
||||
try:
|
||||
wrap_cli._check_and_clear_stale_wrap_marker(path, key="ANTHROPIC_BASE_URL")
|
||||
except BaseException as exc:
|
||||
errors.append(exc)
|
||||
|
||||
def write_new():
|
||||
try:
|
||||
wrap_cli._write_claude_wrap_base_url(
|
||||
"http://127.0.0.1:9300", settings_path=path, port=9300
|
||||
)
|
||||
except BaseException as exc:
|
||||
errors.append(exc)
|
||||
finally:
|
||||
writer_done.set()
|
||||
|
||||
monkeypatch.setattr(wrap_cli, "_read_text", paused_read)
|
||||
recovery = threading.Thread(target=recover, name="stale-recovery")
|
||||
writer = threading.Thread(target=write_new, name="new-writer")
|
||||
recovery.start()
|
||||
try:
|
||||
assert compared.wait(5)
|
||||
writer.start()
|
||||
# Give the competing actual writer the opportunity to commit its marker.
|
||||
writer_done.wait(0.2)
|
||||
finally:
|
||||
resume.set()
|
||||
recovery.join(5)
|
||||
if writer.ident is not None:
|
||||
writer.join(5)
|
||||
assert not recovery.is_alive() and not writer.is_alive()
|
||||
assert not errors, errors
|
||||
assert json.loads(marker_path.read_text(encoding="utf-8"))["port"] == 9300
|
||||
assert (
|
||||
json.loads(path.read_text(encoding="utf-8"))["env"]["ANTHROPIC_BASE_URL"]
|
||||
== "http://127.0.0.1:9300"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user