fix(install): wait for the old process to stop before restarting (#3670)

## Description

`headroom install restart` reports success but leaves the deployment
stopped. Stopping the supervisor returns before the old proxy finishes
its graceful shutdown, so `/readyz` still answers. `_start_deployment`
treats a ready endpoint as "already running" and returns without
starting anything. The old process then exits and nothing replaces it.
`install start` works afterwards because by then nothing is answering.

Restart now waits (bounded, 15s) for the health endpoint to go down
before starting, and fails instead of reporting success if it never
does.

Closes #3658

## Type of Change

- [x] Bug fix (non-breaking change that fixes an issue)
- [ ] New feature (non-breaking change that adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to change)
- [ ] Documentation update
- [ ] Performance improvement
- [ ] Code refactoring (no functional changes)

## Changes Made

- Add `_wait_until_stopped` in `headroom/cli/install.py` and call it
between `_stop_deployment` and `_start_deployment` in `install restart`.
- Add two regression tests: old process lingers after stop (restart must
wait, then start), and old process never stops (restart must fail
without starting or reporting success).

## Testing

- [x] Unit tests pass (`pytest`)
- [x] Linting passes (`ruff check .`)
- [ ] Type checking passes (`mypy headroom`)
- [x] New tests added for new functionality
- [ ] Manual testing performed

### Test Output

```text
python -m pytest tests/test_cli/test_install_cli.py -q
42 passed

python -m ruff check headroom/cli/install.py tests/test_cli/test_install_cli.py -> All checks passed
python -m ruff format --check (same files) -> already formatted

Without the source change, test_install_restart_waits_for_old_process_before_starting fails (1 failed).
```

## Real Behavior Proof

- Environment: Windows 11, Python 3.14.3. The issue was reported on
macOS (launchd), which I could not run.
- Exact command / steps: the unit tests above simulate the old process
still answering `/readyz` after stop.
- Observed result: restart waits for the endpoint to go down, then
starts; it errors if the endpoint never goes down.
- Not tested: a real launchd / systemd restart; mypy; the wider test
suite beyond `test_install_cli.py`; the docker and task presets. The
root cause is inferred from the code path and the issue's symptoms, not
reproduced on macOS.

## Runtime Rollout Safety

- Rollout-managed feature(s): none
- Minimum rollout channel: n/a
- Stable/default behavior changed: `install restart` now waits up to 15s
for the old process to stop, and errors if it does not.
- Kill switch / disable path: n/a
- Unsafe override required: no
- Qualification impact: none
- Rollback path: revert this commit

## Review Readiness

- [x] I have performed a self-review
- [x] This PR is ready for human review

## Checklist

- [x] My code follows the project's style guidelines
- [x] I have performed a self-review of my code
- [x] I have commented my code, particularly in hard-to-understand areas
- [ ] I have made corresponding changes to the documentation
- [x] My changes generate no new warnings
- [x] I have added tests that prove my fix is effective or that my
feature works
- [ ] New and existing unit tests pass locally with my changes
- [x] I did **not** edit `CHANGELOG.md` — it is generated by
release-please from my Conventional Commit PR title (a CI guard enforces
this)

## Additional Notes

Docs: not applicable (no user-facing docs describe restart timing). Full
unit suite not run locally; only `test_install_cli.py`.
This commit is contained in:
Radhakrishnan Pachyappan
2026-09-24 11:29:03 -05:00
committed by GitHub
parent f3d95ee275
commit 11c7320bbc
4 changed files with 141 additions and 0 deletions
+10
View File
@@ -33,6 +33,7 @@ from headroom.install.runtime import (
start_persistent_docker,
stop_runtime,
wait_ready,
wait_stopped,
)
from headroom.install.state import (
ManifestError,
@@ -191,6 +192,15 @@ def _stop_deployment(manifest: DeploymentManifest) -> None:
if manifest.supervisor_kind == SupervisorKind.SERVICE.value:
stop_supervisor(manifest)
stop_runtime(manifest)
# Stopping returns before the old process has finished shutting down, so it
# can keep answering /readyz. `_start_deployment` treats a ready endpoint as
# "already running" and would skip the start, leaving the deployment stopped
# once the old process exits. Block until it is really gone.
if not wait_stopped(manifest):
raise click.ClickException(
f"Deployment '{manifest.profile}' is still answering on "
f"{manifest.health_url} after stop."
)
def _deactivate_deployment_mutations(
+11
View File
@@ -395,6 +395,17 @@ def wait_ready(manifest: DeploymentManifest, timeout_seconds: int = 30) -> bool:
return False
def wait_stopped(manifest: DeploymentManifest, timeout_seconds: int = 15) -> bool:
"""Wait for the deployment's health endpoint to stop answering."""
deadline = time.monotonic() + timeout_seconds
while probe_ready(manifest.health_url):
if time.monotonic() >= deadline:
return False
time.sleep(0.25)
return True
def runtime_status(manifest: DeploymentManifest) -> str:
"""Return a short status string for the deployment runtime."""
+77
View File
@@ -543,6 +543,7 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:
)
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: calls.append("save"))
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
monkeypatch.setattr("headroom.cli.install.wait_stopped", lambda manifest: True)
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")
result = runner.invoke(main, ["install", "restart"])
@@ -560,6 +561,82 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:
]
def _restart_race_manifest():
class Manifest:
profile = "default"
preset = "persistent-service"
runtime_kind = "python"
supervisor_kind = "service"
scope = "user"
health_url = "http://127.0.0.1:8787/readyz"
mutations: list = []
return Manifest()
def test_install_restart_waits_for_old_process_before_starting(monkeypatch) -> None:
"""#3658: the old proxy still answers /readyz right after stop; restart must
wait for it to go away, otherwise start is skipped and the service stays down."""
runner = CliRunner()
calls: list[str] = []
monkeypatch.setattr(
"headroom.cli.install.load_manifest", lambda profile: _restart_race_manifest()
)
monkeypatch.setattr(
"headroom.cli.install.stop_supervisor", lambda manifest: calls.append("stop_supervisor")
)
monkeypatch.setattr(
"headroom.cli.install.stop_runtime", lambda manifest: calls.append("stop_runtime")
)
monkeypatch.setattr(
"headroom.cli.install.wait_stopped",
lambda manifest, timeout_seconds=15: calls.append("wait_stopped") or True,
)
monkeypatch.setattr(
"headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")
)
monkeypatch.setattr(
"headroom.cli.install.wait_ready", lambda manifest, timeout_seconds=45: True
)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda manifest: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: None)
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")
result = runner.invoke(main, ["install", "restart"])
assert result.exit_code == 0, result.output
assert calls == ["stop_supervisor", "stop_runtime", "wait_stopped", "start_supervisor"]
@pytest.mark.parametrize("command", ["stop", "restart"])
def test_install_stop_and_restart_fail_when_old_process_never_stops(monkeypatch, command) -> None:
runner = CliRunner()
calls: list[str] = []
monkeypatch.setattr(
"headroom.cli.install.load_manifest", lambda profile: _restart_race_manifest()
)
monkeypatch.setattr("headroom.cli.install.stop_supervisor", lambda manifest: None)
monkeypatch.setattr("headroom.cli.install.stop_runtime", lambda manifest: None)
monkeypatch.setattr(
"headroom.cli.install.wait_stopped", lambda manifest, timeout_seconds=15: False
)
monkeypatch.setattr(
"headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")
)
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: None)
result = runner.invoke(main, ["install", command])
assert result.exit_code != 0
assert "still answering" in result.output
assert "deployment 'default'." not in result.output
assert calls == []
def test_install_start_noops_when_already_healthy(monkeypatch) -> None:
runner = CliRunner()
calls: list[str] = []
+43
View File
@@ -26,9 +26,52 @@ from headroom.install.runtime import (
start_persistent_docker,
stop_runtime,
wait_ready,
wait_stopped,
)
def _wait_stopped_manifest() -> DeploymentManifest:
return DeploymentManifest(
profile="default",
preset="persistent-service",
runtime_kind="python",
supervisor_kind="service",
scope="user",
provider_mode="manual",
targets=[],
port=8787,
host="127.0.0.1",
backend="anthropic",
health_url="http://127.0.0.1:8787/readyz",
)
def test_wait_stopped_waits_for_health_endpoint_to_go_down(monkeypatch) -> None:
"""#3658: the old process keeps answering /readyz briefly after it is stopped."""
probe_results = iter([True, True, False])
sleeps: list[float] = []
monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: next(probe_results))
monkeypatch.setattr(
"headroom.install.runtime.time.sleep", lambda seconds: sleeps.append(seconds)
)
assert wait_stopped(_wait_stopped_manifest(), timeout_seconds=5) is True
assert len(sleeps) == 2
def test_wait_stopped_times_out_when_endpoint_keeps_answering(monkeypatch) -> None:
clock = {"now": 0.0}
monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: True)
monkeypatch.setattr("headroom.install.runtime.time.monotonic", lambda: clock["now"])
monkeypatch.setattr(
"headroom.install.runtime.time.sleep",
lambda seconds: clock.update(now=clock["now"] + seconds),
)
assert wait_stopped(_wait_stopped_manifest(), timeout_seconds=2) is False
def test_build_runtime_command_for_docker_includes_deployment_env(
monkeypatch, tmp_path: Path
) -> None: