mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-16 05:20:22 +08:00
John T. Myers
1d3b741ee3
* feat(providers): support sandbox provider attach lifecycle Closes #1171 Adds sandbox provider list, attach, and detach API/CLI support while keeping provider policy and credential resolution derived from current sandbox attachments. * fix(providers): refresh sandbox provider credentials Adds provider environment revisions and generation-scoped sandbox credential snapshots so future SSH and exec launches pick up provider attach, detach, and credential updates without mutating already-running processes. Also blocks provider deletion while attached to prevent stale sandbox provider references. * fix(providers): serialize sandbox object mutations * test(providers): cover sandbox provider attach lifecycle * test(providers): accept versioned credential placeholders
501 lines
17 KiB
Python
501 lines
17 KiB
Python
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
"""E2E tests for supervisor-managed provider placeholders in sandboxes.
|
|
|
|
Provider credentials are fetched at runtime by the sandbox supervisor via the
|
|
GetSandboxProviderEnvironment gRPC call. Sandboxed child processes should see
|
|
placeholder values (not raw secrets). Credentials must never be present in the
|
|
persisted sandbox spec environment map.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import time
|
|
from contextlib import contextmanager
|
|
from typing import TYPE_CHECKING
|
|
|
|
import grpc
|
|
import pytest
|
|
|
|
from openshell._proto import datamodel_pb2, openshell_pb2, sandbox_pb2
|
|
|
|
if TYPE_CHECKING:
|
|
from collections.abc import Callable, Iterator
|
|
|
|
from openshell import Sandbox, SandboxClient
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Policy helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _is_placeholder_for_env_key(value: str, key: str) -> bool:
|
|
"""Return true when value is an OpenShell credential placeholder for key."""
|
|
prefix = "openshell:resolve:env:"
|
|
if value == f"{prefix}{key}":
|
|
return True
|
|
token = value.removeprefix(prefix)
|
|
if token == value:
|
|
return False
|
|
return token.startswith("v") and token.endswith(f"_{key}")
|
|
|
|
|
|
def _default_policy() -> sandbox_pb2.SandboxPolicy:
|
|
"""Build a sandbox policy with standard filesystem/process/landlock settings."""
|
|
return sandbox_pb2.SandboxPolicy(
|
|
version=1,
|
|
filesystem=sandbox_pb2.FilesystemPolicy(
|
|
include_workdir=True,
|
|
read_only=["/usr", "/lib", "/etc", "/app", "/dev/urandom"],
|
|
read_write=["/sandbox", "/tmp"],
|
|
),
|
|
landlock=sandbox_pb2.LandlockPolicy(compatibility="best_effort"),
|
|
process=sandbox_pb2.ProcessPolicy(
|
|
run_as_user="sandbox", run_as_group="sandbox"
|
|
),
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Provider lifecycle helper
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@contextmanager
|
|
def provider(
|
|
stub: object,
|
|
*,
|
|
name: str,
|
|
provider_type: str,
|
|
credentials: dict[str, str],
|
|
) -> Iterator[str]:
|
|
"""Create a provider for the duration of the block, then delete it."""
|
|
_delete_provider(stub, name)
|
|
stub.CreateProvider(
|
|
openshell_pb2.CreateProviderRequest(
|
|
provider=datamodel_pb2.Provider(
|
|
metadata=datamodel_pb2.ObjectMeta(name=name),
|
|
type=provider_type,
|
|
credentials=credentials,
|
|
)
|
|
)
|
|
)
|
|
try:
|
|
yield name
|
|
finally:
|
|
_delete_provider(stub, name)
|
|
|
|
|
|
def _delete_provider(stub: object, name: str) -> None:
|
|
"""Delete a provider, ignoring not-found errors."""
|
|
try:
|
|
stub.DeleteProvider(openshell_pb2.DeleteProviderRequest(name=name))
|
|
except grpc.RpcError as exc:
|
|
if hasattr(exc, "code") and exc.code() == grpc.StatusCode.NOT_FOUND:
|
|
pass
|
|
else:
|
|
raise
|
|
|
|
|
|
# ===========================================================================
|
|
# Tests: placeholder visibility
|
|
# ===========================================================================
|
|
|
|
|
|
def test_provider_credentials_available_as_env_vars(
|
|
sandbox: Callable[..., Sandbox],
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""Sandbox child processes see provider env vars as placeholders."""
|
|
with provider(
|
|
sandbox_client._stub,
|
|
name="e2e-test-provider-env",
|
|
provider_type="claude",
|
|
credentials={"ANTHROPIC_API_KEY": "sk-e2e-test-key-12345"},
|
|
) as provider_name:
|
|
spec = datamodel_pb2.SandboxSpec(
|
|
policy=_default_policy(),
|
|
providers=[provider_name],
|
|
)
|
|
|
|
def read_env_var() -> str:
|
|
import os
|
|
|
|
return os.environ.get("ANTHROPIC_API_KEY", "NOT_SET")
|
|
|
|
with sandbox(spec=spec, delete_on_exit=True) as sb:
|
|
result = sb.exec_python(read_env_var)
|
|
assert result.exit_code == 0, result.stderr
|
|
value = result.stdout.strip()
|
|
assert _is_placeholder_for_env_key(value, "ANTHROPIC_API_KEY")
|
|
assert value != "sk-e2e-test-key-12345"
|
|
|
|
|
|
def test_generic_provider_credentials_available_as_env_vars(
|
|
sandbox: Callable[..., Sandbox],
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""Generic provider env vars are placeholders, not raw secrets."""
|
|
with provider(
|
|
sandbox_client._stub,
|
|
name="e2e-test-generic-provider-env",
|
|
provider_type="generic",
|
|
credentials={
|
|
"CUSTOM_SERVICE_TOKEN": "token-generic-123",
|
|
"CUSTOM_SERVICE_URL": "https://internal.example.test/api",
|
|
},
|
|
) as provider_name:
|
|
spec = datamodel_pb2.SandboxSpec(
|
|
policy=_default_policy(),
|
|
providers=[provider_name],
|
|
)
|
|
|
|
def read_generic_env_vars() -> str:
|
|
import os
|
|
|
|
token = os.environ.get("CUSTOM_SERVICE_TOKEN", "NOT_SET")
|
|
url = os.environ.get("CUSTOM_SERVICE_URL", "NOT_SET")
|
|
return f"{token}|{url}"
|
|
|
|
with sandbox(spec=spec, delete_on_exit=True) as sb:
|
|
result = sb.exec_python(read_generic_env_vars)
|
|
assert result.exit_code == 0, result.stderr
|
|
token, url = result.stdout.strip().split("|")
|
|
assert _is_placeholder_for_env_key(token, "CUSTOM_SERVICE_TOKEN")
|
|
assert _is_placeholder_for_env_key(url, "CUSTOM_SERVICE_URL")
|
|
|
|
|
|
def test_nvidia_provider_injects_nvidia_api_key_env_var(
|
|
sandbox: Callable[..., Sandbox],
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""NVIDIA provider projects a placeholder env value into child processes."""
|
|
with provider(
|
|
sandbox_client._stub,
|
|
name="e2e-test-nvidia-provider-env",
|
|
provider_type="nvidia",
|
|
credentials={"NVIDIA_API_KEY": "nvapi-e2e-test-key"},
|
|
) as provider_name:
|
|
spec = datamodel_pb2.SandboxSpec(
|
|
policy=_default_policy(),
|
|
providers=[provider_name],
|
|
)
|
|
|
|
def read_nvidia_key() -> str:
|
|
import os
|
|
|
|
return os.environ.get("NVIDIA_API_KEY", "NOT_SET")
|
|
|
|
with sandbox(spec=spec, delete_on_exit=True) as sb:
|
|
result = sb.exec_python(read_nvidia_key)
|
|
assert result.exit_code == 0, result.stderr
|
|
assert _is_placeholder_for_env_key(
|
|
result.stdout.strip(), "NVIDIA_API_KEY"
|
|
)
|
|
|
|
|
|
def test_attach_detach_updates_credentials_for_later_exec_launches(
|
|
sandbox: Callable[..., Sandbox],
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""Later exec launches see provider attach/detach credential changes."""
|
|
stub = sandbox_client._stub
|
|
provider_name = "e2e-test-attach-detach-env"
|
|
|
|
with provider(
|
|
stub,
|
|
name=provider_name,
|
|
provider_type="generic",
|
|
credentials={"CUSTOM_ATTACH_TOKEN": "token-attach-detach"},
|
|
):
|
|
spec = datamodel_pb2.SandboxSpec(policy=_default_policy(), providers=[])
|
|
|
|
def read_attach_token() -> str:
|
|
import os
|
|
|
|
return os.environ.get("CUSTOM_ATTACH_TOKEN", "NOT_SET")
|
|
|
|
def exec_token(sb: Sandbox) -> str:
|
|
result = sb.exec_python(read_attach_token)
|
|
assert result.exit_code == 0, result.stderr
|
|
return result.stdout.strip()
|
|
|
|
def wait_for_token(sb: Sandbox, expected: str) -> None:
|
|
deadline = time.monotonic() + 35
|
|
last = None
|
|
while time.monotonic() < deadline:
|
|
last = exec_token(sb)
|
|
if expected == "NOT_SET":
|
|
matched = last == expected
|
|
else:
|
|
matched = _is_placeholder_for_env_key(last, "CUSTOM_ATTACH_TOKEN")
|
|
if matched:
|
|
return
|
|
time.sleep(2)
|
|
pytest.fail(f"expected {expected!r}, last exec saw {last!r}")
|
|
|
|
with sandbox(spec=spec, delete_on_exit=True) as sb:
|
|
assert exec_token(sb) == "NOT_SET"
|
|
|
|
try:
|
|
stub.AttachSandboxProvider(
|
|
openshell_pb2.AttachSandboxProviderRequest(
|
|
sandbox_name=sb.sandbox.name,
|
|
provider_name=provider_name,
|
|
)
|
|
)
|
|
wait_for_token(
|
|
sb,
|
|
"openshell:resolve:env:CUSTOM_ATTACH_TOKEN",
|
|
)
|
|
|
|
stub.DetachSandboxProvider(
|
|
openshell_pb2.DetachSandboxProviderRequest(
|
|
sandbox_name=sb.sandbox.name,
|
|
provider_name=provider_name,
|
|
)
|
|
)
|
|
wait_for_token(sb, "NOT_SET")
|
|
finally:
|
|
try:
|
|
stub.DetachSandboxProvider(
|
|
openshell_pb2.DetachSandboxProviderRequest(
|
|
sandbox_name=sb.sandbox.name,
|
|
provider_name=provider_name,
|
|
)
|
|
)
|
|
except grpc.RpcError as exc:
|
|
if exc.code() != grpc.StatusCode.NOT_FOUND:
|
|
raise
|
|
|
|
|
|
# ===========================================================================
|
|
# Tests: security & edge cases
|
|
# ===========================================================================
|
|
|
|
|
|
def test_ssh_handshake_secret_not_visible_in_exec_environment(
|
|
sandbox: Callable[..., Sandbox],
|
|
) -> None:
|
|
def read_handshake_secret() -> str:
|
|
import os
|
|
|
|
return os.environ.get("OPENSHELL_SSH_HANDSHAKE_SECRET", "NOT_SET")
|
|
|
|
with sandbox(delete_on_exit=True) as sb:
|
|
result = sb.exec_python(read_handshake_secret)
|
|
assert result.exit_code == 0, result.stderr
|
|
assert result.stdout.strip() == "NOT_SET"
|
|
|
|
|
|
def test_create_sandbox_rejects_unknown_provider(
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""CreateSandbox fails fast when a provider name does not exist."""
|
|
spec = datamodel_pb2.SandboxSpec(
|
|
policy=_default_policy(),
|
|
providers=["nonexistent-provider-xyz"],
|
|
)
|
|
with pytest.raises(grpc.RpcError) as exc_info:
|
|
sandbox_client.create(spec=spec)
|
|
|
|
assert exc_info.value.code() == grpc.StatusCode.FAILED_PRECONDITION
|
|
assert "nonexistent-provider-xyz" in (exc_info.value.details() or "")
|
|
|
|
|
|
def test_credentials_not_in_persisted_spec_environment(
|
|
sandbox: Callable[..., Sandbox],
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""Provider credentials should NOT appear in the sandbox spec's environment map."""
|
|
with provider(
|
|
sandbox_client._stub,
|
|
name="e2e-test-no-persist",
|
|
provider_type="claude",
|
|
credentials={"ANTHROPIC_API_KEY": "sk-should-not-persist"},
|
|
) as provider_name:
|
|
spec = datamodel_pb2.SandboxSpec(
|
|
policy=_default_policy(),
|
|
providers=[provider_name],
|
|
)
|
|
|
|
with sandbox(spec=spec, delete_on_exit=True) as sb:
|
|
fetched = sandbox_client._stub.GetSandbox(
|
|
openshell_pb2.GetSandboxRequest(name=sb.sandbox.name)
|
|
)
|
|
persisted_env = dict(fetched.sandbox.spec.environment)
|
|
assert "ANTHROPIC_API_KEY" not in persisted_env, (
|
|
"credentials should not be persisted in sandbox spec environment"
|
|
)
|
|
|
|
|
|
# ===========================================================================
|
|
# Tests: provider update merge semantics
|
|
# ===========================================================================
|
|
|
|
|
|
def test_update_provider_preserves_unset_credentials_and_config(
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""Updating one credential must not clobber other credentials or config."""
|
|
stub = sandbox_client._stub
|
|
name = "merge-test-preserve"
|
|
_delete_provider(stub, name)
|
|
|
|
try:
|
|
stub.CreateProvider(
|
|
openshell_pb2.CreateProviderRequest(
|
|
provider=datamodel_pb2.Provider(
|
|
metadata=datamodel_pb2.ObjectMeta(name=name),
|
|
type="generic",
|
|
credentials={"KEY_A": "val-a", "KEY_B": "val-b"},
|
|
config={"BASE_URL": "https://example.com"},
|
|
)
|
|
)
|
|
)
|
|
|
|
stub.UpdateProvider(
|
|
openshell_pb2.UpdateProviderRequest(
|
|
provider=datamodel_pb2.Provider(
|
|
metadata=datamodel_pb2.ObjectMeta(name=name),
|
|
type="",
|
|
credentials={"KEY_A": "rotated-a"},
|
|
)
|
|
)
|
|
)
|
|
|
|
got = stub.GetProvider(openshell_pb2.GetProviderRequest(name=name))
|
|
p = got.provider
|
|
# Credential keys are preserved but values are redacted.
|
|
assert len(p.credentials) > 0, "credential keys should be preserved"
|
|
for key, val in p.credentials.items():
|
|
assert val == "REDACTED", (
|
|
f"credential '{key}' should be REDACTED, got '{val}'"
|
|
)
|
|
assert p.config["BASE_URL"] == "https://example.com", (
|
|
"config should be preserved"
|
|
)
|
|
finally:
|
|
_delete_provider(stub, name)
|
|
|
|
|
|
def test_update_provider_empty_maps_preserves_all(
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""Sending empty credential and config maps should be a no-op."""
|
|
stub = sandbox_client._stub
|
|
name = "merge-test-noop"
|
|
_delete_provider(stub, name)
|
|
|
|
try:
|
|
stub.CreateProvider(
|
|
openshell_pb2.CreateProviderRequest(
|
|
provider=datamodel_pb2.Provider(
|
|
metadata=datamodel_pb2.ObjectMeta(name=name),
|
|
type="generic",
|
|
credentials={"TOKEN": "secret"},
|
|
config={"URL": "https://api.example.com"},
|
|
)
|
|
)
|
|
)
|
|
|
|
stub.UpdateProvider(
|
|
openshell_pb2.UpdateProviderRequest(
|
|
provider=datamodel_pb2.Provider(
|
|
metadata=datamodel_pb2.ObjectMeta(name=name),
|
|
type="",
|
|
)
|
|
)
|
|
)
|
|
|
|
got = stub.GetProvider(openshell_pb2.GetProviderRequest(name=name))
|
|
p = got.provider
|
|
# Credential keys are preserved but values are redacted.
|
|
assert len(p.credentials) > 0, "credential keys should be preserved"
|
|
for key, val in p.credentials.items():
|
|
assert val == "REDACTED", (
|
|
f"credential '{key}' should be REDACTED, got '{val}'"
|
|
)
|
|
assert p.config["URL"] == "https://api.example.com"
|
|
finally:
|
|
_delete_provider(stub, name)
|
|
|
|
|
|
def test_update_provider_merges_config_preserves_credentials(
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""Updating only config should not touch credentials."""
|
|
stub = sandbox_client._stub
|
|
name = "merge-test-config-only"
|
|
_delete_provider(stub, name)
|
|
|
|
try:
|
|
stub.CreateProvider(
|
|
openshell_pb2.CreateProviderRequest(
|
|
provider=datamodel_pb2.Provider(
|
|
metadata=datamodel_pb2.ObjectMeta(name=name),
|
|
type="generic",
|
|
credentials={"API_KEY": "original-key"},
|
|
config={"ENDPOINT": "https://old.example.com"},
|
|
)
|
|
)
|
|
)
|
|
|
|
stub.UpdateProvider(
|
|
openshell_pb2.UpdateProviderRequest(
|
|
provider=datamodel_pb2.Provider(
|
|
metadata=datamodel_pb2.ObjectMeta(name=name),
|
|
type="",
|
|
config={"ENDPOINT": "https://new.example.com"},
|
|
)
|
|
)
|
|
)
|
|
|
|
got = stub.GetProvider(openshell_pb2.GetProviderRequest(name=name))
|
|
p = got.provider
|
|
# Credential keys are preserved but values are redacted.
|
|
assert len(p.credentials) > 0, "credential keys should be preserved"
|
|
for key, val in p.credentials.items():
|
|
assert val == "REDACTED", (
|
|
f"credential '{key}' should be REDACTED, got '{val}'"
|
|
)
|
|
assert p.config["ENDPOINT"] == "https://new.example.com"
|
|
finally:
|
|
_delete_provider(stub, name)
|
|
|
|
|
|
def test_update_provider_rejects_type_change(
|
|
sandbox_client: SandboxClient,
|
|
) -> None:
|
|
"""Attempting to change a provider's type must be rejected."""
|
|
stub = sandbox_client._stub
|
|
name = "merge-test-type-reject"
|
|
_delete_provider(stub, name)
|
|
|
|
try:
|
|
stub.CreateProvider(
|
|
openshell_pb2.CreateProviderRequest(
|
|
provider=datamodel_pb2.Provider(
|
|
metadata=datamodel_pb2.ObjectMeta(name=name),
|
|
type="generic",
|
|
credentials={"KEY": "val"},
|
|
)
|
|
)
|
|
)
|
|
|
|
with pytest.raises(grpc.RpcError) as exc_info:
|
|
stub.UpdateProvider(
|
|
openshell_pb2.UpdateProviderRequest(
|
|
provider=datamodel_pb2.Provider(
|
|
metadata=datamodel_pb2.ObjectMeta(name=name),
|
|
type="nvidia",
|
|
)
|
|
)
|
|
)
|
|
assert exc_info.value.code() == grpc.StatusCode.INVALID_ARGUMENT
|
|
assert "type cannot be changed" in exc_info.value.details()
|
|
finally:
|
|
_delete_provider(stub, name)
|