feat(inference): validate endpoints before saving routes

Closes #273

Verify inference endpoints synchronously on the server during set/update, expose a --no-verify escape hatch in the CLI and Python helper, and return actionable failures when validation does not pass.
This commit is contained in:
Piotr Mlocek
2026-03-15 16:39:35 -07:00
parent 53b7ce7112
commit 5f31de631b
10 changed files with 511 additions and 21 deletions
+2
View File
@@ -398,11 +398,13 @@ class InferenceRouteClient:
*,
provider_name: str,
model_id: str,
no_verify: bool = False,
) -> ClusterInferenceConfig:
response = self._stub.SetClusterInference(
inference_pb2.SetClusterInferenceRequest(
provider_name=provider_name,
model_id=model_id,
skip_validation=no_verify,
),
timeout=self._timeout,
)
+34
View File
@@ -10,6 +10,7 @@ from openshell._proto import openshell_pb2
from openshell.sandbox import (
_PYTHON_CLOUDPICKLE_BOOTSTRAP,
_SANDBOX_PYTHON_BIN,
InferenceRouteClient,
SandboxClient,
)
@@ -33,6 +34,22 @@ class _FakeStub:
)
class _FakeInferenceStub:
def __init__(self) -> None:
self.request = None
def SetClusterInference(self, request: Any, timeout: float | None = None) -> Any:
self.request = request
_ = timeout
class _Response:
provider_name = request.provider_name
model_id = request.model_id
version = 1
return _Response()
def _client_with_fake_stub(stub: _FakeStub) -> SandboxClient:
client = cast("SandboxClient", object.__new__(SandboxClient))
client._timeout = 30.0
@@ -120,3 +137,20 @@ def test_from_active_cluster_prefers_openshell_gateway_env(
assert client._cluster_name == gateway_name
finally:
client.close()
def test_inference_set_cluster_forwards_no_verify_flag() -> None:
stub = _FakeInferenceStub()
client = cast("InferenceRouteClient", object.__new__(InferenceRouteClient))
client._timeout = 30.0
client._stub = cast("Any", stub)
result = client.set_cluster(
provider_name="openai-dev",
model_id="gpt-4.1",
no_verify=True,
)
assert result.provider_name == "openai-dev"
assert stub.request is not None
assert stub.request.skip_validation is True