// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 #![cfg(feature = "e2e")] //! E2E test: verify Kubernetes user namespace pod spec generation. //! //! Enables `OPENSHELL_ENABLE_USER_NAMESPACES` on the gateway, triggers sandbox //! creation, and inspects the resulting pod spec to confirm: //! 1. `spec.hostUsers` is `false` //! 2. The container security context requests no added capabilities and //! drops every capability //! //! The sandbox pod may fail to start in Docker-in-Docker dev clusters where the //! filesystem does not support ID-mapped mounts. The test inspects the pod spec //! regardless of runtime success. use std::process::Stdio; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use openshell_e2e::harness::binary::openshell_cmd; use tokio::process::Child; async fn kubectl(args: &[&str]) -> Result { let output = tokio::process::Command::new("docker") .args(["exec", "openshell-cluster-openshell", "kubectl"]) .args(args) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .output() .await .map_err(|e| format!("failed to run kubectl: {e}"))?; let stdout = String::from_utf8_lossy(&output.stdout).to_string(); let stderr = String::from_utf8_lossy(&output.stderr).to_string(); if !output.status.success() { return Err(format!("kubectl {args:?} failed: {stdout}{stderr}")); } Ok(stdout) } async fn set_user_namespaces(enable: bool) -> Result<(), String> { let env_arg = if enable { "OPENSHELL_ENABLE_USER_NAMESPACES=true" } else { "OPENSHELL_ENABLE_USER_NAMESPACES-" }; kubectl(&[ "set", "env", "statefulset/openshell", "-n", "openshell", env_arg, ]) .await?; kubectl(&[ "rollout", "status", "statefulset/openshell", "-n", "openshell", "--timeout=120s", ]) .await?; // Give the gateway time to fully initialize after rollout. tokio::time::sleep(Duration::from_secs(5)).await; Ok(()) } async fn delete_sandbox(name: &str) { let _ = kubectl(&["delete", "sandbox", name, "-n", "openshell"]).await; } fn unique_sandbox_name() -> String { let suffix = SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_millis(); format!("userns-e2e-{suffix}") } async fn stop_child(child: &mut Child) { let _ = child.kill().await; let _ = child.wait().await; } async fn wait_for_sandbox(name: &str, timeout_secs: u64) -> Result<(), String> { let deadline = tokio::time::Instant::now() + Duration::from_secs(timeout_secs); while tokio::time::Instant::now() < deadline { if let Ok(n) = kubectl(&[ "get", "sandbox", name, "-n", "openshell", "-o", "jsonpath={.metadata.name}", ]) .await { if !n.trim().is_empty() { return Ok(()); } } tokio::time::sleep(Duration::from_secs(2)).await; } Err(format!( "sandbox {name} did not appear within {timeout_secs}s" )) } /// Find a sandbox pod by its sandbox CRD name. The CRD controller creates a /// pod with the same name as the Sandbox resource. async fn wait_for_sandbox_pod(name: &str, timeout_secs: u64) -> Result<(), String> { let deadline = tokio::time::Instant::now() + Duration::from_secs(timeout_secs); while tokio::time::Instant::now() < deadline { if let Ok(n) = kubectl(&[ "get", "pod", name, "-n", "openshell", "-o", "jsonpath={.metadata.name}", ]) .await { if !n.trim().is_empty() { return Ok(()); } } tokio::time::sleep(Duration::from_secs(2)).await; } Err(format!( "sandbox pod {name} did not appear within {timeout_secs}s" )) } // Disabled by default — not reachable from any project-controlled cluster // and brittle by design. See https://github.com/NVIDIA/OpenShell/issues/1597 // for the tracking issue context. Re-enable with `cargo test -- --ignored` // after the issues below are addressed. // // Blocking issues: // 1. `kubectl` is invoked as `docker exec openshell-cluster-openshell kubectl`. // No setup in the repo (helm-k3s-local.sh, e2e/with-kube-gateway.sh, the // CI kind workflow) creates a docker container with that name; only an // external OpenShift/manual setup matches it. // 2. The test mutates the gateway StatefulSet via `kubectl set env`, which // triggers a pod rollout mid-test. The wrapper's `kubectl port-forward` // to the old pod is disrupted during the rollout, and the test's // `openshell sandbox create` is spawned without capturing stderr, so // transient connection failures surface as a generic "sandbox did not // appear within 60s" with no actionable signal. #[ignore = "broken: hardcoded docker exec container name + brittle mid-test gateway rollout (see header)"] #[tokio::test] async fn sandbox_pod_spec_has_user_namespace_fields() { // Enable user namespaces on the gateway. set_user_namespaces(true) .await .expect("failed to enable user namespaces on gateway"); let sandbox_name = unique_sandbox_name(); // Start sandbox creation in the background. The pod may never become // ready in DinD environments, so we spawn the CLI and inspect the pod // spec independently. let mut cmd = openshell_cmd(); cmd.arg("sandbox") .arg("create") .arg("--name") .arg(&sandbox_name) .arg("--") .arg("sleep") .arg("infinity"); cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); let mut child = cmd.spawn().expect("failed to spawn openshell create"); if let Err(e) = wait_for_sandbox(&sandbox_name, 60).await { stop_child(&mut child).await; delete_sandbox(&sandbox_name).await; set_user_namespaces(false).await.ok(); panic!("{e}"); } // Wait for the pod to be created (the CRD controller creates it). if let Err(e) = wait_for_sandbox_pod(&sandbox_name, 60).await { stop_child(&mut child).await; delete_sandbox(&sandbox_name).await; set_user_namespaces(false).await.ok(); panic!("{e}"); } // Inspect the pod spec for hostUsers. let host_users = kubectl(&[ "get", "pod", &sandbox_name, "-n", "openshell", "-o", "jsonpath={.spec.hostUsers}", ]) .await; // Inspect capabilities on the agent container. let cap_add = kubectl(&[ "get", "pod", &sandbox_name, "-n", "openshell", "-o", "jsonpath={.spec.containers[?(@.name=='agent')].securityContext.capabilities.add}", ]) .await; let cap_drop = kubectl(&[ "get", "pod", &sandbox_name, "-n", "openshell", "-o", "jsonpath={.spec.containers[?(@.name=='agent')].securityContext.capabilities.drop}", ]) .await; // Clean up. stop_child(&mut child).await; delete_sandbox(&sandbox_name).await; set_user_namespaces(false).await.ok(); // Assert hostUsers is false. let host_users_val = host_users.expect("failed to get hostUsers from pod spec"); assert_eq!( host_users_val.trim(), "false", "sandbox pod must have spec.hostUsers=false when user namespaces are enabled" ); let cap_add = cap_add.expect("failed to get added capabilities from pod spec"); assert!( cap_add.trim().is_empty(), "sandbox pod must not request added capabilities, got: {cap_add}" ); let cap_drop = cap_drop.expect("failed to get dropped capabilities from pod spec"); assert!( cap_drop.contains("ALL"), "sandbox pod must drop every capability, got: {cap_drop}" ); }