mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(install): wait for the old process to stop before restarting (#3670)
## Description `headroom install restart` reports success but leaves the deployment stopped. Stopping the supervisor returns before the old proxy finishes its graceful shutdown, so `/readyz` still answers. `_start_deployment` treats a ready endpoint as "already running" and returns without starting anything. The old process then exits and nothing replaces it. `install start` works afterwards because by then nothing is answering. Restart now waits (bounded, 15s) for the health endpoint to go down before starting, and fails instead of reporting success if it never does. Closes #3658 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature (non-breaking change that adds functionality) - [ ] Breaking change (fix or feature that would cause existing functionality to change) - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring (no functional changes) ## Changes Made - Add `_wait_until_stopped` in `headroom/cli/install.py` and call it between `_stop_deployment` and `_start_deployment` in `install restart`. - Add two regression tests: old process lingers after stop (restart must wait, then start), and old process never stops (restart must fail without starting or reporting success). ## Testing - [x] Unit tests pass (`pytest`) - [x] Linting passes (`ruff check .`) - [ ] Type checking passes (`mypy headroom`) - [x] New tests added for new functionality - [ ] Manual testing performed ### Test Output ```text python -m pytest tests/test_cli/test_install_cli.py -q 42 passed python -m ruff check headroom/cli/install.py tests/test_cli/test_install_cli.py -> All checks passed python -m ruff format --check (same files) -> already formatted Without the source change, test_install_restart_waits_for_old_process_before_starting fails (1 failed). ``` ## Real Behavior Proof - Environment: Windows 11, Python 3.14.3. The issue was reported on macOS (launchd), which I could not run. - Exact command / steps: the unit tests above simulate the old process still answering `/readyz` after stop. - Observed result: restart waits for the endpoint to go down, then starts; it errors if the endpoint never goes down. - Not tested: a real launchd / systemd restart; mypy; the wider test suite beyond `test_install_cli.py`; the docker and task presets. The root cause is inferred from the code path and the issue's symptoms, not reproduced on macOS. ## Runtime Rollout Safety - Rollout-managed feature(s): none - Minimum rollout channel: n/a - Stable/default behavior changed: `install restart` now waits up to 15s for the old process to stop, and errors if it does not. - Kill switch / disable path: n/a - Unsafe override required: no - Qualification impact: none - Rollback path: revert this commit ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review of my code - [x] I have commented my code, particularly in hard-to-understand areas - [ ] I have made corresponding changes to the documentation - [x] My changes generate no new warnings - [x] I have added tests that prove my fix is effective or that my feature works - [ ] New and existing unit tests pass locally with my changes - [x] I did **not** edit `CHANGELOG.md` — it is generated by release-please from my Conventional Commit PR title (a CI guard enforces this) ## Additional Notes Docs: not applicable (no user-facing docs describe restart timing). Full unit suite not run locally; only `test_install_cli.py`.
This commit is contained in:
@@ -33,6 +33,7 @@ from headroom.install.runtime import (
|
||||
start_persistent_docker,
|
||||
stop_runtime,
|
||||
wait_ready,
|
||||
wait_stopped,
|
||||
)
|
||||
from headroom.install.state import (
|
||||
ManifestError,
|
||||
@@ -191,6 +192,15 @@ def _stop_deployment(manifest: DeploymentManifest) -> None:
|
||||
if manifest.supervisor_kind == SupervisorKind.SERVICE.value:
|
||||
stop_supervisor(manifest)
|
||||
stop_runtime(manifest)
|
||||
# Stopping returns before the old process has finished shutting down, so it
|
||||
# can keep answering /readyz. `_start_deployment` treats a ready endpoint as
|
||||
# "already running" and would skip the start, leaving the deployment stopped
|
||||
# once the old process exits. Block until it is really gone.
|
||||
if not wait_stopped(manifest):
|
||||
raise click.ClickException(
|
||||
f"Deployment '{manifest.profile}' is still answering on "
|
||||
f"{manifest.health_url} after stop."
|
||||
)
|
||||
|
||||
|
||||
def _deactivate_deployment_mutations(
|
||||
|
||||
@@ -395,6 +395,17 @@ def wait_ready(manifest: DeploymentManifest, timeout_seconds: int = 30) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def wait_stopped(manifest: DeploymentManifest, timeout_seconds: int = 15) -> bool:
|
||||
"""Wait for the deployment's health endpoint to stop answering."""
|
||||
|
||||
deadline = time.monotonic() + timeout_seconds
|
||||
while probe_ready(manifest.health_url):
|
||||
if time.monotonic() >= deadline:
|
||||
return False
|
||||
time.sleep(0.25)
|
||||
return True
|
||||
|
||||
|
||||
def runtime_status(manifest: DeploymentManifest) -> str:
|
||||
"""Return a short status string for the deployment runtime."""
|
||||
|
||||
|
||||
@@ -543,6 +543,7 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:
|
||||
)
|
||||
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: calls.append("save"))
|
||||
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
|
||||
monkeypatch.setattr("headroom.cli.install.wait_stopped", lambda manifest: True)
|
||||
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")
|
||||
|
||||
result = runner.invoke(main, ["install", "restart"])
|
||||
@@ -560,6 +561,82 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:
|
||||
]
|
||||
|
||||
|
||||
def _restart_race_manifest():
|
||||
class Manifest:
|
||||
profile = "default"
|
||||
preset = "persistent-service"
|
||||
runtime_kind = "python"
|
||||
supervisor_kind = "service"
|
||||
scope = "user"
|
||||
health_url = "http://127.0.0.1:8787/readyz"
|
||||
mutations: list = []
|
||||
|
||||
return Manifest()
|
||||
|
||||
|
||||
def test_install_restart_waits_for_old_process_before_starting(monkeypatch) -> None:
|
||||
"""#3658: the old proxy still answers /readyz right after stop; restart must
|
||||
wait for it to go away, otherwise start is skipped and the service stays down."""
|
||||
|
||||
runner = CliRunner()
|
||||
calls: list[str] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"headroom.cli.install.load_manifest", lambda profile: _restart_race_manifest()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom.cli.install.stop_supervisor", lambda manifest: calls.append("stop_supervisor")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom.cli.install.stop_runtime", lambda manifest: calls.append("stop_runtime")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom.cli.install.wait_stopped",
|
||||
lambda manifest, timeout_seconds=15: calls.append("wait_stopped") or True,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom.cli.install.wait_ready", lambda manifest, timeout_seconds=45: True
|
||||
)
|
||||
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda manifest: [])
|
||||
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: None)
|
||||
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
|
||||
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")
|
||||
|
||||
result = runner.invoke(main, ["install", "restart"])
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert calls == ["stop_supervisor", "stop_runtime", "wait_stopped", "start_supervisor"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("command", ["stop", "restart"])
|
||||
def test_install_stop_and_restart_fail_when_old_process_never_stops(monkeypatch, command) -> None:
|
||||
runner = CliRunner()
|
||||
calls: list[str] = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
"headroom.cli.install.load_manifest", lambda profile: _restart_race_manifest()
|
||||
)
|
||||
monkeypatch.setattr("headroom.cli.install.stop_supervisor", lambda manifest: None)
|
||||
monkeypatch.setattr("headroom.cli.install.stop_runtime", lambda manifest: None)
|
||||
monkeypatch.setattr(
|
||||
"headroom.cli.install.wait_stopped", lambda manifest, timeout_seconds=15: False
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")
|
||||
)
|
||||
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: None)
|
||||
|
||||
result = runner.invoke(main, ["install", command])
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert "still answering" in result.output
|
||||
assert "deployment 'default'." not in result.output
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_install_start_noops_when_already_healthy(monkeypatch) -> None:
|
||||
runner = CliRunner()
|
||||
calls: list[str] = []
|
||||
|
||||
@@ -26,9 +26,52 @@ from headroom.install.runtime import (
|
||||
start_persistent_docker,
|
||||
stop_runtime,
|
||||
wait_ready,
|
||||
wait_stopped,
|
||||
)
|
||||
|
||||
|
||||
def _wait_stopped_manifest() -> DeploymentManifest:
|
||||
return DeploymentManifest(
|
||||
profile="default",
|
||||
preset="persistent-service",
|
||||
runtime_kind="python",
|
||||
supervisor_kind="service",
|
||||
scope="user",
|
||||
provider_mode="manual",
|
||||
targets=[],
|
||||
port=8787,
|
||||
host="127.0.0.1",
|
||||
backend="anthropic",
|
||||
health_url="http://127.0.0.1:8787/readyz",
|
||||
)
|
||||
|
||||
|
||||
def test_wait_stopped_waits_for_health_endpoint_to_go_down(monkeypatch) -> None:
|
||||
"""#3658: the old process keeps answering /readyz briefly after it is stopped."""
|
||||
|
||||
probe_results = iter([True, True, False])
|
||||
sleeps: list[float] = []
|
||||
monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: next(probe_results))
|
||||
monkeypatch.setattr(
|
||||
"headroom.install.runtime.time.sleep", lambda seconds: sleeps.append(seconds)
|
||||
)
|
||||
|
||||
assert wait_stopped(_wait_stopped_manifest(), timeout_seconds=5) is True
|
||||
assert len(sleeps) == 2
|
||||
|
||||
|
||||
def test_wait_stopped_times_out_when_endpoint_keeps_answering(monkeypatch) -> None:
|
||||
clock = {"now": 0.0}
|
||||
monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: True)
|
||||
monkeypatch.setattr("headroom.install.runtime.time.monotonic", lambda: clock["now"])
|
||||
monkeypatch.setattr(
|
||||
"headroom.install.runtime.time.sleep",
|
||||
lambda seconds: clock.update(now=clock["now"] + seconds),
|
||||
)
|
||||
|
||||
assert wait_stopped(_wait_stopped_manifest(), timeout_seconds=2) is False
|
||||
|
||||
|
||||
def test_build_runtime_command_for_docker_includes_deployment_env(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
|
||||
Reference in New Issue
Block a user