// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 #![cfg(feature = "e2e-kubernetes-workspace-managed")] //! E2E tests for managed workspace mode. //! //! The gateway is deployed with `workspace_mode = "managed"`, which //! auto-creates a K8s namespace per workspace (`openshell-{gateway_id}-{ws}`) //! and deletes it when the last sandbox is removed. //! //! Namespace cleanup after sandbox deletion is best-effort and depends on //! controller finalization timing. These tests focus on verifiable behavior: //! namespace creation, labels, ServiceAccount and SSH NetworkPolicy //! provisioning, and sandbox CR placement in the correct namespace. use std::process::Stdio; use std::time::Duration; use openshell_e2e::harness::binary::{openshell_bin, openshell_cmd}; use openshell_e2e::harness::output::strip_ansi; const DURABLE_MAIN_SCRIPT: &str = r#"echo "$1"; exec sleep infinity"#; fn kube_context() -> String { std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE") .expect("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE must be set") } async fn kubectl(args: &[&str]) -> (bool, String) { let context = kube_context(); let output = tokio::process::Command::new("kubectl") .arg("--context") .arg(&context) .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .output() .await .expect("failed to spawn kubectl"); let combined = format!( "{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), ); (output.status.success(), combined) } fn managed_namespace(workspace: &str) -> String { format!("openshell-openshell-{workspace}") } async fn run_cli(args: &[&str]) -> (bool, String) { let mut cmd = openshell_cmd(); cmd.args(args).stdout(Stdio::piped()).stderr(Stdio::piped()); let output = cmd.output().await.expect("failed to spawn openshell"); let combined = format!( "{}{}", String::from_utf8_lossy(&output.stdout), String::from_utf8_lossy(&output.stderr), ); (output.status.success(), strip_ansi(&combined)) } fn unique_workspace(prefix: &str) -> String { let ts = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap_or_default() .as_millis() % 100_000; format!("{prefix}-{ts}") } async fn wait_sandbox_gone(workspace: &str, sandbox: &str) { let deadline = tokio::time::Instant::now() + Duration::from_secs(30); loop { let (ok, out) = run_cli(&["sandbox", "list", "--workspace", workspace]).await; if ok && !out.contains(sandbox) { return; } if tokio::time::Instant::now() >= deadline { panic!( "sandbox {sandbox} still listed in workspace {workspace} 30s after delete: {out}" ); } tokio::time::sleep(Duration::from_secs(2)).await; } } struct ManagedCleanup { workspace: String, sandboxes: Vec, } impl Drop for ManagedCleanup { fn drop(&mut self) { let bin = openshell_bin(); for sb in &self.sandboxes { let _ = std::process::Command::new(&bin) .args(["sandbox", "delete", sb, "--workspace", &self.workspace]) .stdout(Stdio::null()) .stderr(Stdio::null()) .status(); } let _ = std::process::Command::new(&bin) .args(["workspace", "delete", &self.workspace]) .stdout(Stdio::null()) .stderr(Stdio::null()) .status(); let context = std::env::var("OPENSHELL_E2E_KUBE_CONTEXT_ACTIVE").unwrap_or_default(); if !context.is_empty() { let ns = managed_namespace(&self.workspace); let _ = std::process::Command::new("kubectl") .args([ "--context", &context, "delete", "namespace", &ns, "--ignore-not-found", "--wait=false", ]) .stdout(Stdio::null()) .stderr(Stdio::null()) .status(); } } } #[tokio::test] async fn managed_creates_namespace_with_labels() { let ws = unique_workspace("mgd"); let ns = managed_namespace(&ws); let _cleanup = ManagedCleanup { workspace: ws.clone(), sandboxes: vec!["mgd-sb".into()], }; let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; assert!(ok, "workspace create failed: {out}"); // Create a sandbox — this triggers namespace creation. let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "mgd-sb", "--", "echo", "managed-ok", ]) .await; assert!(ok, "sandbox create failed: {out}"); assert!( out.contains("managed-ok"), "sandbox output missing expected string: {out}" ); // Verify the managed namespace was created. let (ok, out) = kubectl(&["get", "namespace", &ns]).await; assert!(ok, "managed namespace {ns} should exist: {out}"); // Verify labels on the namespace. let (ok, label_out) = kubectl(&["get", "namespace", &ns, "-o", "jsonpath={.metadata.labels}"]).await; assert!(ok, "failed to read namespace labels: {label_out}"); assert!( label_out.contains("openshell.ai/managed-by"), "namespace missing managed-by label: {label_out}" ); assert!( label_out.contains("openshell.ai/gateway-id"), "namespace missing gateway-id label: {label_out}" ); // Verify the ServiceAccount was created in the managed namespace. let (ok, _) = kubectl(&["get", "serviceaccount", "openshell-sandbox", "-n", &ns]).await; assert!(ok, "ServiceAccount openshell-sandbox should exist in {ns}"); // Each runtime generation gets its own immutable copy of the configured // image-pull Secret; the configured name is never created in {ns}. let (ok, staged) = kubectl(&[ "get", "secret", "-n", &ns, "-l", "openshell.ai/component=image-pull", "-o", "jsonpath={range .items[*]}{.metadata.name} {.type} {.immutable}{\"\\n\"}{end}", ]) .await; assert!( ok && staged.contains("os-pull-") && staged.contains("kubernetes.io/dockerconfigjson true"), "a generation image-pull Secret should exist in {ns}: {staged}" ); let (ok, _) = kubectl(&["get", "secret", "e2e-regcred", "-n", &ns]).await; assert!( !ok, "configured image-pull Secret name must not exist in {ns}" ); // Verify SSH ingress is restricted to the gateway peer. Because Kubernetes // NetworkPolicies are allowlists, the absence of a sandbox peer here // denies sandbox-to-sandbox TCP 2222 traffic. let (ok, policy) = kubectl(&[ "get", "networkpolicy", "openshell-sandbox-ssh", "-n", &ns, "-o", "json", ]) .await; assert!( ok, "managed SSH NetworkPolicy should exist in {ns}: {policy}" ); let policy: serde_json::Value = serde_json::from_str(&policy).expect("managed SSH NetworkPolicy should be valid JSON"); assert_eq!( policy["spec"]["podSelector"]["matchLabels"]["openshell.ai/managed-by"], "openshell" ); assert_eq!(policy["spec"]["ingress"][0]["ports"][0]["port"], 2222); assert_eq!( policy["spec"]["ingress"][0]["from"][0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"], "openshell" ); assert!( policy["spec"]["ingress"][0]["from"][0]["podSelector"]["matchLabels"] ["app.kubernetes.io/name"] .is_string(), "SSH ingress peer must select gateway pods: {policy}" ); // Verify sandbox CR is in the managed namespace (not the gateway namespace). let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CR should exist in namespace {ns}: {out}"); assert!(out.contains("mgd-sb"), "sandbox CR name mismatch: {out}"); // Verify the sandbox is resolvable through the OpenShell control plane. let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; assert!(ok, "sandbox list failed: {out}"); assert!( out.contains("mgd-sb"), "sandbox list should find mgd-sb via control plane: {out}" ); let (ok, out) = run_cli(&["sandbox", "get", "mgd-sb", "--workspace", &ws]).await; assert!(ok, "sandbox get failed: {out}"); assert!( out.contains("mgd-sb"), "sandbox get should resolve mgd-sb via control plane: {out}" ); // Verify sandbox delete works through the control plane (uses sandbox_lookup_selector). let (ok, out) = run_cli(&["sandbox", "delete", "mgd-sb", "--workspace", &ws]).await; assert!(ok, "sandbox delete failed: {out}"); // Wait for the sandbox CR to be fully removed (deletion is asynchronous). wait_sandbox_gone(&ws, "mgd-sb").await; } #[tokio::test] async fn managed_namespace_survives_with_remaining_sandboxes() { let ws = unique_workspace("mgd2"); let ns = managed_namespace(&ws); let _cleanup = ManagedCleanup { workspace: ws.clone(), sandboxes: vec!["sb-a".into(), "sb-b".into()], }; let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; assert!(ok, "workspace create failed: {out}"); // Create two sandboxes. let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "sb-a", "--", "echo", "a", ]) .await; assert!(ok, "sandbox sb-a create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "sb-b", "--", "echo", "b", ]) .await; assert!(ok, "sandbox sb-b create failed: {out}"); // Delete first sandbox — namespace should survive because sb-b still exists. let (ok, out) = run_cli(&["sandbox", "delete", "sb-a", "--workspace", &ws]).await; assert!(ok, "sandbox sb-a delete failed: {out}"); // Wait for sb-a to be fully removed before checking namespace state. wait_sandbox_gone(&ws, "sb-a").await; let (ok, _) = kubectl(&["get", "namespace", &ns]).await; assert!(ok, "managed namespace {ns} should still exist with sb-b"); // Verify sb-b's CR is still in the managed namespace. let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns, "-o", "name"]).await; assert!(ok, "sandbox CRs should still exist in {ns}: {out}"); assert!( out.contains("sb-b"), "sb-b CR should still be present: {out}" ); // Verify sb-b is still resolvable through the OpenShell control plane. let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws]).await; assert!(ok, "sandbox list failed: {out}"); assert!( out.contains("sb-b"), "sandbox list should find sb-b via control plane: {out}" ); } #[tokio::test] async fn managed_isolates_workspaces_into_separate_namespaces() { let ws_a = unique_workspace("iso-a"); let ws_b = unique_workspace("iso-b"); let ns_a = managed_namespace(&ws_a); let ns_b = managed_namespace(&ws_b); let _cleanup_a = ManagedCleanup { workspace: ws_a.clone(), sandboxes: vec!["sb-iso-a".into()], }; let _cleanup_b = ManagedCleanup { workspace: ws_b.clone(), sandboxes: vec!["sb-iso-b".into()], }; // Create two workspaces with sandboxes. let (ok, out) = run_cli(&["workspace", "create", "--name", &ws_a]).await; assert!(ok, "workspace A create failed: {out}"); let (ok, out) = run_cli(&["workspace", "create", "--name", &ws_b]).await; assert!(ok, "workspace B create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws_a, "--name", "sb-iso-a", "--", "echo", "a", ]) .await; assert!(ok, "sandbox A create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws_b, "--name", "sb-iso-b", "--", "echo", "b", ]) .await; assert!(ok, "sandbox B create failed: {out}"); // Verify each workspace has its own namespace. assert_ne!(ns_a, ns_b, "namespaces should differ"); let (ok, _) = kubectl(&["get", "namespace", &ns_a]).await; assert!(ok, "namespace {ns_a} should exist"); let (ok, _) = kubectl(&["get", "namespace", &ns_b]).await; assert!(ok, "namespace {ns_b} should exist"); // Verify sandbox CRs are in the correct namespaces (no cross-contamination). let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns_a, "-o", "name"]).await; assert!(ok, "failed to list CRs in {ns_a}: {out}"); assert!(out.contains("sb-iso-a"), "sb-iso-a should be in {ns_a}"); assert!( !out.contains("sb-iso-b"), "sb-iso-b should NOT be in {ns_a}" ); let (ok, out) = kubectl(&["get", "sandbox.agents.x-k8s.io", "-n", &ns_b, "-o", "name"]).await; assert!(ok, "failed to list CRs in {ns_b}: {out}"); assert!(out.contains("sb-iso-b"), "sb-iso-b should be in {ns_b}"); assert!( !out.contains("sb-iso-a"), "sb-iso-a should NOT be in {ns_b}" ); // Verify workspace isolation through the OpenShell control plane. let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws_a]).await; assert!(ok, "sandbox list ws_a failed: {out}"); assert!( out.contains("sb-iso-a"), "sandbox list ws_a should find sb-iso-a: {out}" ); assert!( !out.contains("sb-iso-b"), "sandbox list ws_a should NOT find sb-iso-b: {out}" ); let (ok, out) = run_cli(&["sandbox", "list", "--workspace", &ws_b]).await; assert!(ok, "sandbox list ws_b failed: {out}"); assert!( out.contains("sb-iso-b"), "sandbox list ws_b should find sb-iso-b: {out}" ); assert!( !out.contains("sb-iso-a"), "sandbox list ws_b should NOT find sb-iso-a: {out}" ); } #[tokio::test] async fn managed_workspace_delete_removes_namespace() { let ws = unique_workspace("mgddel"); let ns = managed_namespace(&ws); let _cleanup = ManagedCleanup { workspace: ws.clone(), sandboxes: vec!["del-sb".into()], }; let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; assert!(ok, "workspace create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "del-sb", "--", "echo", "del-ok", ]) .await; assert!(ok, "sandbox create failed: {out}"); assert!( out.contains("del-ok"), "sandbox output missing expected string: {out}" ); let (ok, _) = kubectl(&["get", "namespace", &ns]).await; assert!( ok, "managed namespace {ns} should exist after sandbox create" ); let (ok, out) = run_cli(&["sandbox", "delete", "del-sb", "--workspace", &ws]).await; assert!(ok, "sandbox delete failed: {out}"); wait_sandbox_gone(&ws, "del-sb").await; let (ok, out) = run_cli(&["workspace", "delete", &ws]).await; assert!(ok, "workspace delete failed: {out}"); let deadline = tokio::time::Instant::now() + Duration::from_secs(30); loop { let (exists, _) = kubectl(&["get", "namespace", &ns]).await; if !exists { break; } if tokio::time::Instant::now() >= deadline { panic!("managed namespace {ns} still exists 30s after workspace delete"); } tokio::time::sleep(Duration::from_secs(2)).await; } } #[tokio::test] async fn managed_supervisor_reads_client_tls_from_its_bootstrap_secret() { let (ok, config_out) = kubectl(&[ "get", "configmap", "openshell-config", "-n", "openshell", "-o", "jsonpath={.data.gateway\\.toml}", ]) .await; if !ok || !config_out.contains("client_tls_secret_name") { eprintln!("SKIP: client_tls_secret_name not configured"); return; } let ws = unique_workspace("mgdtls"); let ns = managed_namespace(&ws); let _cleanup = ManagedCleanup { workspace: ws.clone(), sandboxes: vec!["tls-sb".into()], }; let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; assert!(ok, "workspace create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "tls-sb", "--", "echo", "tls-ok", ]) .await; assert!(ok, "sandbox create failed: {out}"); assert!( out.contains("tls-ok"), "sandbox output missing expected string: {out}" ); let (ok, _) = kubectl(&["get", "secret", "openshell-client-tls", "-n", &ns]).await; assert!( !ok, "client TLS Secret must not be copied into managed namespace {ns}" ); let (ok, keys) = kubectl(&[ "get", "secret", "-n", &ns, "-l", "openshell.ai/component=supervisor-bootstrap", "-o", "jsonpath={.items[*].data}", ]) .await; assert!( ok && keys.contains("client-tls.crt") && keys.contains("client-ca.crt"), "supervisor bootstrap Secret should carry client TLS material: {keys}" ); } const GATEWAY_SERVICE_ACCOUNT: &str = "system:serviceaccount:openshell:openshell"; async fn gateway_can(verb: &str, resource: &str, namespace: &str) -> bool { let (_, out) = kubectl(&[ "auth", "can-i", verb, resource, "-n", namespace, "--as", GATEWAY_SERVICE_ACCOUNT, ]) .await; match out.trim() { "yes" => true, "no" => false, other => panic!("unexpected kubectl auth can-i output for {verb} {resource}: {other}"), } } #[tokio::test] async fn managed_gateway_cannot_read_or_modify_workspace_secrets() { let ns = managed_namespace(&unique_workspace("mgdrbac")); for verb in ["get", "patch"] { assert!( !gateway_can(verb, "secrets", &ns).await, "gateway must not {verb} arbitrary Secrets in workspace namespace {ns}" ); assert!( !gateway_can(verb, "secrets/unrelated-secret", &ns).await, "gateway must not {verb} an unrelated Secret in workspace namespace {ns}" ); } assert!( !gateway_can("list", "secrets", &ns).await, "gateway must not list Secrets in workspace namespace {ns}" ); for copied in ["secrets/openshell-client-tls", "secrets/e2e-regcred"] { assert!( !gateway_can("get", copied, &ns).await, "gateway must not read {copied} outside its sandbox namespace" ); } assert!( gateway_can("get", "secrets", "openshell").await, "gateway must still reach provider credential Secrets in its configured namespace" ); } #[tokio::test] async fn managed_rejects_namespace_owned_by_different_gateway() { let ws = unique_workspace("mgdown"); let ns = managed_namespace(&ws); let _cleanup = ManagedCleanup { workspace: ws.clone(), sandboxes: vec![], }; let (ok, out) = kubectl(&["create", "namespace", &ns]).await; assert!(ok, "failed to pre-create namespace {ns}: {out}"); let (ok, out) = kubectl(&[ "label", "namespace", &ns, "openshell.ai/managed-by=openshell", "openshell.ai/gateway-id=wrong-gateway", ]) .await; assert!(ok, "failed to label namespace: {out}"); let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; assert!(ok, "workspace create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "conflict-sb", "--", "echo", "nope", ]) .await; assert!( !ok, "sandbox create should fail for namespace owned by different gateway, but succeeded: {out}" ); } #[tokio::test] async fn managed_full_lifecycle_with_multiple_sandboxes() { let ws = unique_workspace("mgdlc"); let ns = managed_namespace(&ws); let _cleanup = ManagedCleanup { workspace: ws.clone(), sandboxes: vec!["lc-a".into(), "lc-b".into()], }; let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; assert!(ok, "workspace create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "lc-a", "--", "echo", "a", ]) .await; assert!(ok, "sandbox lc-a create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "lc-b", "--", "echo", "b", ]) .await; assert!(ok, "sandbox lc-b create failed: {out}"); let (ok, _) = kubectl(&["get", "namespace", &ns]).await; assert!(ok, "managed namespace {ns} should exist"); let (ok, out) = run_cli(&["sandbox", "delete", "lc-a", "--workspace", &ws]).await; assert!(ok, "sandbox lc-a delete failed: {out}"); wait_sandbox_gone(&ws, "lc-a").await; let (ok, _) = kubectl(&["get", "namespace", &ns]).await; assert!( ok, "managed namespace {ns} should still exist with lc-b remaining" ); let (ok, out) = run_cli(&["sandbox", "delete", "lc-b", "--workspace", &ws]).await; assert!(ok, "sandbox lc-b delete failed: {out}"); wait_sandbox_gone(&ws, "lc-b").await; let (ok, out) = run_cli(&["workspace", "delete", &ws]).await; assert!(ok, "workspace delete failed: {out}"); let deadline = tokio::time::Instant::now() + Duration::from_secs(30); loop { let (exists, _) = kubectl(&["get", "namespace", &ns]).await; if !exists { break; } if tokio::time::Instant::now() >= deadline { panic!("managed namespace {ns} still exists 30s after full lifecycle cleanup"); } tokio::time::sleep(Duration::from_secs(2)).await; } } #[tokio::test] async fn managed_stop_waits_for_workspace_pod_to_disappear() { let (ok, sandbox_api_version) = kubectl(&[ "get", "crd", "sandboxes.agents.x-k8s.io", "-o", "jsonpath={.spec.versions[?(@.storage==true)].name}", ]) .await; assert!( ok, "failed to resolve Sandbox API version: {sandbox_api_version}" ); if sandbox_api_version.trim() != "v1alpha1" { eprintln!("SKIP: legacy pod-disappearance fallback applies only to Sandbox API v1alpha1"); return; } let ws = unique_workspace("mgdstop"); let ns = managed_namespace(&ws); let sandbox = "stop-sb"; let _cleanup = ManagedCleanup { workspace: ws.clone(), sandboxes: vec![sandbox.into()], }; let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; assert!(ok, "workspace create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", sandbox, "--detach", "--", "sh", "-c", DURABLE_MAIN_SCRIPT, "_", "ready", ]) .await; assert!(ok, "sandbox create failed: {out}"); let (ok, pod_name) = kubectl(&[ "get", "sandbox", sandbox, "-n", &ns, "-o", "jsonpath={.metadata.annotations.agents\\.x-k8s\\.io/pod-name}", ]) .await; assert!(ok, "failed to resolve sandbox pod name: {pod_name}"); let pod_name = pod_name.trim(); assert!(!pod_name.is_empty(), "sandbox pod annotation was empty"); let (ok, out) = run_cli(&["sandbox", "stop", sandbox, "--workspace", &ws]).await; assert!(ok, "sandbox stop failed: {out}"); let (exists, out) = kubectl(&["get", "pod", pod_name, "-n", &ns]).await; assert!( !exists, "sandbox stop returned before workspace pod {pod_name} disappeared: {out}" ); } #[tokio::test] async fn managed_rejects_invalid_dns1123_sandbox_name() { let ws = unique_workspace("mgddns"); let _cleanup = ManagedCleanup { workspace: ws.clone(), sandboxes: vec![], }; let (ok, out) = run_cli(&["workspace", "create", "--name", &ws]).await; assert!(ok, "workspace create failed: {out}"); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "my_bad_name", "--", "echo", "nope", ]) .await; assert!( !ok, "sandbox with underscore name should be rejected: {out}" ); assert!( out.contains("lowercase alphanumeric"), "error should mention character constraint: {out}" ); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "MyBadName", "--", "echo", "nope", ]) .await; assert!(!ok, "sandbox with uppercase name should be rejected: {out}"); assert!( out.contains("lowercase alphanumeric"), "error should mention character constraint: {out}" ); let (ok, out) = run_cli(&[ "sandbox", "create", "--workspace", &ws, "--name", "trailing-", "--", "echo", "nope", ]) .await; assert!( !ok, "sandbox with trailing hyphen should be rejected: {out}" ); let normalized: String = out .chars() .filter(|c| *c != '│') .collect::() .split_whitespace() .collect::>() .join(" "); assert!( normalized.contains("must not start or end with a hyphen"), "error should mention hyphen constraint: {out}" ); }