mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-26 06:52:57 +08:00
refactor(kubernetes): describe the single runtime placement
Signed-off-by: Drew Newberry <anewberry@nvidia.com>
This commit is contained in:
Drew Newberry
@@ -70,7 +70,8 @@ mise run helm:skaffold:run
|
||||
```
|
||||
|
||||
Both commands build the gateway and trusted runtime image and deploy the
|
||||
OpenShell Helm chart. Kubernetes sandboxes always use the proxy-pod topology:
|
||||
OpenShell Helm chart. The Kubernetes driver creates a sandbox workload Pod and a
|
||||
separate supervisor Deployment:
|
||||
a capability-free sandbox Pod behind an empty-egress NetworkPolicy and a
|
||||
separate capability-free supervisor Deployment. The default Skaffold values
|
||||
export gateway and Kubernetes-driver traces to the collector service installed
|
||||
|
||||
@@ -406,12 +406,10 @@ jobs:
|
||||
include:
|
||||
- agent_sandbox_api: v1beta1
|
||||
agent_sandbox_version: v0.5.0
|
||||
topology: proxy-pod
|
||||
extra_helm_values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
extra_helm_values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml
|
||||
- agent_sandbox_api: v1alpha1
|
||||
agent_sandbox_version: v0.4.6
|
||||
topology: proxy-pod
|
||||
extra_helm_values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
extra_helm_values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
@@ -419,7 +417,7 @@ jobs:
|
||||
uses: ./.github/workflows/e2e-kubernetes-test.yml
|
||||
with:
|
||||
image-tag: ${{ github.sha }}
|
||||
job-name: Kubernetes E2E (Rust smoke, ${{ matrix.topology }}, Agent Sandbox ${{ matrix.agent_sandbox_api }})
|
||||
job-name: Kubernetes E2E (Rust smoke, Agent Sandbox ${{ matrix.agent_sandbox_api }})
|
||||
agent-sandbox-version: ${{ matrix.agent_sandbox_version }}
|
||||
extra-helm-values: ${{ matrix.extra_helm_values }}
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
@@ -436,7 +434,7 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
job-name: Kubernetes E2E (workspace managed mode)
|
||||
e2e-task: e2e:kubernetes:workspace-managed
|
||||
extra-helm-values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
extra-helm-values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-external-driver-e2e:
|
||||
@@ -454,7 +452,7 @@ jobs:
|
||||
gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
|
||||
external-driver-binary: openshell-driver-kubernetes
|
||||
cluster-images: sandbox supervisor
|
||||
extra-helm-values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
extra-helm-values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-workspace-operator-e2e:
|
||||
@@ -469,7 +467,7 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
job-name: Kubernetes E2E (workspace operator mode)
|
||||
e2e-task: e2e:kubernetes:workspace-operator
|
||||
extra-helm-values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
extra-helm-values: deploy/helm/openshell/ci/values-sandbox-runtime.yaml
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-ha-e2e:
|
||||
|
||||
@@ -171,7 +171,7 @@ impl ProviderCredentialState {
|
||||
/// Build a static provider state from an already-prepared child
|
||||
/// environment snapshot.
|
||||
///
|
||||
/// The Kubernetes proxy-pod topology uses this in the sandbox process:
|
||||
/// The Kubernetes sandbox runtime uses this in the sandbox process:
|
||||
/// the supervisor owns provider credential resolvers and sends the
|
||||
/// workload-facing env map over a local control channel. The process leaf
|
||||
/// must inject that map into child processes without re-placeholderizing it
|
||||
|
||||
@@ -133,9 +133,9 @@ pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED";
|
||||
|
||||
/// The isolation backend admitted by the deployment configuration (RFC 0012).
|
||||
///
|
||||
/// Delivered on a channel separate from the topology descriptor so descriptor
|
||||
/// Delivered on a channel separate from the backend descriptor so descriptor
|
||||
/// verification against the admitted backend is not self-referential. Required
|
||||
/// whenever a topology descriptor is supplied.
|
||||
/// whenever a backend descriptor is supplied.
|
||||
pub const ADMITTED_ISOLATION_BACKEND: &str = "OPENSHELL_ADMITTED_ISOLATION_BACKEND";
|
||||
|
||||
/// Comma-separated runtime networking capabilities supplied by the compute
|
||||
|
||||
@@ -67,7 +67,7 @@ The driver creates an empty-egress `NetworkPolicy` before it releases the
|
||||
workload Pod. A second rule permits only the paired supervisor Pod to reach the
|
||||
sandbox TLS port. The supervisor has normal egress for gateway, DNS, and
|
||||
policy-approved upstream connections. Set
|
||||
`proxy_pod.network_policy_enforced = true` only after verifying that the cluster
|
||||
`sandbox_runtime.network_policy_enforced = true` only after verifying that the cluster
|
||||
CNI enforces ingress and egress `NetworkPolicy` for sandbox namespaces.
|
||||
|
||||
Each sandbox generation uses two immutable bootstrap Secrets. A trusted init
|
||||
|
||||
@@ -22,10 +22,10 @@ pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default";
|
||||
/// Default storage size for the workspace PVC.
|
||||
pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi";
|
||||
|
||||
/// Driver-owned requirements for the cross-pod RFC 0012 topology.
|
||||
/// Driver-owned requirements for the Kubernetes sandbox runtime.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub struct KubernetesProxyPodConfig {
|
||||
pub struct KubernetesSandboxRuntimeConfig {
|
||||
/// Explicit operator assertion that the cluster CNI enforces
|
||||
/// `networking.k8s.io/v1` `NetworkPolicy` for the sandbox namespaces.
|
||||
pub network_policy_enforced: bool,
|
||||
@@ -33,7 +33,7 @@ pub struct KubernetesProxyPodConfig {
|
||||
pub boundary_port: u16,
|
||||
}
|
||||
|
||||
impl Default for KubernetesProxyPodConfig {
|
||||
impl Default for KubernetesSandboxRuntimeConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
network_policy_enforced: false,
|
||||
@@ -42,16 +42,16 @@ impl Default for KubernetesProxyPodConfig {
|
||||
}
|
||||
}
|
||||
|
||||
impl KubernetesProxyPodConfig {
|
||||
impl KubernetesSandboxRuntimeConfig {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if !self.network_policy_enforced {
|
||||
return Err(
|
||||
"proxy-pod topology requires proxy_pod.network_policy_enforced = true after the operator has verified CNI NetworkPolicy enforcement"
|
||||
"sandbox_runtime.network_policy_enforced must be true after the operator has verified CNI NetworkPolicy enforcement"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if self.boundary_port < 1024 {
|
||||
return Err("proxy_pod.boundary_port must be at least 1024".to_string());
|
||||
return Err("sandbox_runtime.boundary_port must be at least 1024".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -132,7 +132,7 @@ pub struct KubernetesComputeConfig {
|
||||
/// operator mode. Hot-reloaded on change. Delivered via `ConfigMap` volume mount.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub operator_namespace_file: Option<String>,
|
||||
/// Kubernetes `ServiceAccount` assigned to both topology Pods. Automatic
|
||||
/// Kubernetes `ServiceAccount` assigned to the workload and supervisor Pods. Automatic
|
||||
/// token mounting is disabled; only the supervisor receives an explicit
|
||||
/// audience-bound projected token accepted by the bootstrap authenticator.
|
||||
pub service_account_name: String,
|
||||
@@ -155,7 +155,7 @@ pub struct KubernetesComputeConfig {
|
||||
/// Empty string delegates to the Kubernetes default.
|
||||
pub supervisor_image_pull_policy: String,
|
||||
/// Cross-pod sandbox/supervisor settings.
|
||||
pub proxy_pod: KubernetesProxyPodConfig,
|
||||
pub sandbox_runtime: KubernetesSandboxRuntimeConfig,
|
||||
/// Corporate HTTP forward proxy used by the network supervisor for
|
||||
/// policy-approved TLS CONNECT egress.
|
||||
pub https_proxy: Option<String>,
|
||||
@@ -269,7 +269,7 @@ impl Default for KubernetesComputeConfig {
|
||||
sandbox_runtime_image_pull_policy: String::new(),
|
||||
supervisor_image: config::default_supervisor_image(),
|
||||
supervisor_image_pull_policy: String::new(),
|
||||
proxy_pod: KubernetesProxyPodConfig::default(),
|
||||
sandbox_runtime: KubernetesSandboxRuntimeConfig::default(),
|
||||
https_proxy: None,
|
||||
no_proxy: None,
|
||||
proxy_auth_secret_name: None,
|
||||
@@ -330,7 +330,7 @@ impl KubernetesComputeConfig {
|
||||
}
|
||||
|
||||
pub fn validate_proxy_uid(&self) -> Result<(), String> {
|
||||
self.proxy_pod.validate()
|
||||
self.sandbox_runtime.validate()
|
||||
}
|
||||
|
||||
/// Validate the operator-owned corporate upstream proxy configuration.
|
||||
@@ -731,7 +731,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_pod_requires_network_policy_enforcement_acknowledgement() {
|
||||
fn sandbox_runtime_requires_network_policy_enforcement_acknowledgement() {
|
||||
let mut cfg = KubernetesComputeConfig::default();
|
||||
assert!(
|
||||
cfg.validate_proxy_uid()
|
||||
@@ -739,7 +739,7 @@ mod tests {
|
||||
.contains("network_policy_enforced")
|
||||
);
|
||||
|
||||
cfg.proxy_pod.network_policy_enforced = true;
|
||||
cfg.sandbox_runtime.network_policy_enforced = true;
|
||||
cfg.validate_proxy_uid().unwrap();
|
||||
}
|
||||
|
||||
@@ -754,19 +754,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("unknown field"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_rejects_removed_topology_alias_field() {
|
||||
let mut json = serde_json::Map::new();
|
||||
json.insert(
|
||||
["supervisor", "topology"].join("_"),
|
||||
serde_json::json!("sidecar"),
|
||||
);
|
||||
let err =
|
||||
serde_json::from_value::<KubernetesComputeConfig>(serde_json::Value::Object(json))
|
||||
.unwrap_err();
|
||||
assert!(err.to_string().contains("unknown field"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_rejects_removed_flat_sidecar_fields() {
|
||||
for json in [
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@
|
||||
//! Kubernetes provisioning for the shared authenticated boundary protocol.
|
||||
//!
|
||||
//! This module deliberately contains no lifecycle, process, network, identity,
|
||||
//! or wire implementation. The driver chooses the proxy-pod placement, binds
|
||||
//! or wire implementation. The driver places the sandbox runtime, binds
|
||||
//! immutable Kubernetes resource identities, and provisions TCP coordinates;
|
||||
//! `openshell-isolation-interface` and `openshell-sandbox` provide the common
|
||||
//! control and boundary behavior.
|
||||
@@ -22,12 +22,12 @@ use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString;
|
||||
use kube::core::ObjectMeta;
|
||||
use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity};
|
||||
use openshell_sandbox_backend::boundary_protocol::{
|
||||
BoundaryConfig, BoundaryListener, BoundaryTopology, GatewayVerificationKey,
|
||||
BoundaryConfig, BoundaryListener, GatewayVerificationKey, SandboxRuntimeDescriptor,
|
||||
SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport,
|
||||
};
|
||||
|
||||
/// Registered RFC 0012 backend name for the proxy-pod topology.
|
||||
pub const BACKEND_NAME: &str = "kubernetes-proxy-pod";
|
||||
/// Isolation backend implemented by the `OpenShell` sandbox runtime.
|
||||
pub const BACKEND_NAME: &str = openshell_sandbox_backend::BACKEND_NAME;
|
||||
|
||||
/// Label that binds the workload and control pods in one unique pair.
|
||||
pub const BOUNDARY_PAIR_LABEL: &str = "openshell.ai/boundary-pair";
|
||||
@@ -40,11 +40,11 @@ const SUPERVISOR_ROLE: &str = "supervisor";
|
||||
|
||||
/// Driver-owned inputs for the workload pod's Kubernetes network fence.
|
||||
///
|
||||
/// This is the first phase of proxy-pod provisioning. The driver applies the
|
||||
/// This is the first phase of sandbox-runtime provisioning. The driver applies the
|
||||
/// returned labels to the respective pods and creates the returned policy. It
|
||||
/// then observes the policy UID and resourceVersion and supplies both to
|
||||
/// `KubernetesProxyPodBoundarySpec`.
|
||||
pub struct KubernetesProxyPodNetworkFenceSpec {
|
||||
/// `KubernetesSandboxRuntimeBoundarySpec`.
|
||||
pub struct KubernetesSandboxRuntimeNetworkFenceSpec {
|
||||
pub namespace: String,
|
||||
pub policy_name: String,
|
||||
/// A unique, Kubernetes-label-safe value generated for this pod pair.
|
||||
@@ -53,13 +53,13 @@ pub struct KubernetesProxyPodNetworkFenceSpec {
|
||||
}
|
||||
|
||||
/// Labels and policy needed to remove direct workload-pod egress.
|
||||
pub struct KubernetesProxyPodNetworkFence {
|
||||
pub struct KubernetesSandboxRuntimeNetworkFence {
|
||||
pub workload_labels: BTreeMap<String, String>,
|
||||
pub control_labels: BTreeMap<String, String>,
|
||||
pub workload_policy: NetworkPolicy,
|
||||
}
|
||||
|
||||
impl KubernetesProxyPodNetworkFenceSpec {
|
||||
impl KubernetesSandboxRuntimeNetworkFenceSpec {
|
||||
/// Render a default-deny workload fence with one control-to-boundary path.
|
||||
///
|
||||
/// Kubernetes `NetworkPolicy` is connection-aware: traffic returning over
|
||||
@@ -67,7 +67,7 @@ impl KubernetesProxyPodNetworkFenceSpec {
|
||||
/// workload pod has no egress rules. The control pod remains responsible
|
||||
/// for opening policy-approved upstream connections.
|
||||
#[must_use]
|
||||
pub fn provision(self) -> KubernetesProxyPodNetworkFence {
|
||||
pub fn provision(self) -> KubernetesSandboxRuntimeNetworkFence {
|
||||
let workload_labels = boundary_pair_labels(&self.pair_label_value, WORKLOAD_ROLE);
|
||||
let control_labels = boundary_pair_labels(&self.pair_label_value, SUPERVISOR_ROLE);
|
||||
|
||||
@@ -107,7 +107,7 @@ impl KubernetesProxyPodNetworkFenceSpec {
|
||||
}),
|
||||
};
|
||||
|
||||
KubernetesProxyPodNetworkFence {
|
||||
KubernetesSandboxRuntimeNetworkFence {
|
||||
workload_labels,
|
||||
control_labels,
|
||||
workload_policy,
|
||||
@@ -126,9 +126,9 @@ fn boundary_pair_labels(pair: &str, role: &str) -> BTreeMap<String, String> {
|
||||
///
|
||||
/// The driver constructs this only after Kubernetes has assigned every UID and
|
||||
/// after it has observed the exact egress policy resource version. The workload
|
||||
/// stays held until the matching boundary config and control topology have been
|
||||
/// stays held until the matching boundary config and supervisor resources have been
|
||||
/// installed.
|
||||
pub struct KubernetesProxyPodBoundarySpec {
|
||||
pub struct KubernetesSandboxRuntimeBoundarySpec {
|
||||
pub boundary_id: String,
|
||||
pub generation: String,
|
||||
pub session_id: openshell_core::SandboxSessionId,
|
||||
@@ -151,17 +151,17 @@ pub struct KubernetesProxyPodBoundarySpec {
|
||||
pub child_env: HashMap<String, String>,
|
||||
}
|
||||
|
||||
/// Protected workload-pod config and matching proxy-pod descriptor.
|
||||
pub struct KubernetesProxyPodBoundaryProvisioning {
|
||||
/// Protected workload-pod config and matching sandbox-runtime descriptor.
|
||||
pub struct KubernetesSandboxRuntimeBoundaryProvisioning {
|
||||
pub boundary_config: BoundaryConfig,
|
||||
pub topology: BoundaryTopology,
|
||||
pub runtime_descriptor: SandboxRuntimeDescriptor,
|
||||
}
|
||||
|
||||
impl KubernetesProxyPodBoundarySpec {
|
||||
impl KubernetesSandboxRuntimeBoundarySpec {
|
||||
/// Produce both sides of the common protocol from one observed Kubernetes
|
||||
/// resource set so a stale or recreated object cannot be attached.
|
||||
#[must_use]
|
||||
pub fn provision(self) -> KubernetesProxyPodBoundaryProvisioning {
|
||||
pub fn provision(self) -> KubernetesSandboxRuntimeBoundaryProvisioning {
|
||||
let resource_claims = BTreeMap::from([
|
||||
("kubernetes.namespace_uid".to_string(), self.namespace_uid),
|
||||
(
|
||||
@@ -193,7 +193,7 @@ impl KubernetesProxyPodBoundarySpec {
|
||||
egress_isolated: true,
|
||||
egress_rule_count: 0,
|
||||
};
|
||||
KubernetesProxyPodBoundaryProvisioning {
|
||||
KubernetesSandboxRuntimeBoundaryProvisioning {
|
||||
boundary_config: BoundaryConfig {
|
||||
boundary_id: self.boundary_id.clone(),
|
||||
generation: self.generation.clone(),
|
||||
@@ -213,7 +213,7 @@ impl KubernetesProxyPodBoundarySpec {
|
||||
driver_fence: driver_fence.clone(),
|
||||
child_env: self.child_env,
|
||||
},
|
||||
topology: BoundaryTopology {
|
||||
runtime_descriptor: SandboxRuntimeDescriptor {
|
||||
boundary_id: self.boundary_id,
|
||||
generation: self.generation,
|
||||
session_id: self.session_id,
|
||||
@@ -235,8 +235,8 @@ impl KubernetesProxyPodBoundarySpec {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn spec() -> KubernetesProxyPodBoundarySpec {
|
||||
KubernetesProxyPodBoundarySpec {
|
||||
fn spec() -> KubernetesSandboxRuntimeBoundarySpec {
|
||||
KubernetesSandboxRuntimeBoundarySpec {
|
||||
boundary_id: "sandbox-1".to_string(),
|
||||
generation: "generation-1".to_string(),
|
||||
session_id: openshell_core::SandboxSessionId::new(),
|
||||
@@ -282,25 +282,25 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
provisioned.boundary_config.resource_claims,
|
||||
provisioned.topology.resource_claims
|
||||
provisioned.runtime_descriptor.resource_claims
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.topology.resource_claims["kubernetes.sandbox_resource_uid"],
|
||||
provisioned.runtime_descriptor.resource_claims["kubernetes.sandbox_resource_uid"],
|
||||
"sandbox-resource-uid"
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.topology.resource_claims["kubernetes.egress_policy_resource_version"],
|
||||
provisioned.runtime_descriptor.resource_claims["kubernetes.egress_policy_resource_version"],
|
||||
"1945"
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.boundary_config.driver_fence,
|
||||
provisioned.topology.driver_fence
|
||||
provisioned.runtime_descriptor.driver_fence
|
||||
);
|
||||
assert!(
|
||||
provisioned
|
||||
.topology
|
||||
.runtime_descriptor
|
||||
.driver_fence
|
||||
.validate_for_backend(BACKEND_NAME)
|
||||
.validate()
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
@@ -320,14 +320,14 @@ mod tests {
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.topology.transport,
|
||||
provisioned.runtime_descriptor.transport,
|
||||
SandboxTransport::Tcp {
|
||||
authority: "os-boundary-sandbox.default.svc:5500".to_string(),
|
||||
addresses: vec!["10.42.0.7:5500".parse().expect("valid target")],
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.topology.tls,
|
||||
provisioned.runtime_descriptor.tls,
|
||||
SandboxTlsClientConfig {
|
||||
server_name: "boundary.sandbox.openshell".to_string(),
|
||||
trust_anchor_pem: "test-ca".to_string(),
|
||||
@@ -337,7 +337,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn network_fence_denies_all_workload_initiated_egress() {
|
||||
let fence = KubernetesProxyPodNetworkFenceSpec {
|
||||
let fence = KubernetesSandboxRuntimeNetworkFenceSpec {
|
||||
namespace: "sandbox-ns".to_string(),
|
||||
policy_name: "openshell-boundary-sandbox-1".to_string(),
|
||||
pair_label_value: "pair-1".to_string(),
|
||||
@@ -359,7 +359,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn network_fence_allows_only_paired_control_to_boundary_port() {
|
||||
let fence = KubernetesProxyPodNetworkFenceSpec {
|
||||
let fence = KubernetesSandboxRuntimeNetworkFenceSpec {
|
||||
namespace: "sandbox-ns".to_string(),
|
||||
policy_name: "openshell-boundary-sandbox-1".to_string(),
|
||||
pair_label_value: "pair-1".to_string(),
|
||||
|
||||
@@ -6,12 +6,12 @@ pub mod driver;
|
||||
pub mod grpc;
|
||||
pub mod isolation;
|
||||
pub mod otel_tracing;
|
||||
mod proxy_pod;
|
||||
mod sandbox_runtime;
|
||||
|
||||
pub use config::{
|
||||
DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE,
|
||||
KubernetesComputeConfig, KubernetesProxyPodConfig, ManagedSshIngressConfig, WorkspaceMode,
|
||||
managed_namespace_prefix,
|
||||
KubernetesComputeConfig, KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig,
|
||||
WorkspaceMode, managed_namespace_prefix,
|
||||
};
|
||||
pub use driver::{KubernetesComputeDriver, KubernetesDriverError};
|
||||
pub use grpc::ComputeDriverService;
|
||||
|
||||
@@ -12,7 +12,7 @@ use openshell_core::VERSION;
|
||||
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
|
||||
use openshell_driver_kubernetes::{
|
||||
ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME,
|
||||
KubernetesComputeConfig, KubernetesComputeDriver, KubernetesProxyPodConfig,
|
||||
KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSandboxRuntimeConfig,
|
||||
ManagedSshIngressConfig, WorkspaceMode,
|
||||
};
|
||||
|
||||
@@ -123,17 +123,17 @@ struct Args {
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_K8S_PROXY_POD_NETWORK_POLICY_ENFORCED",
|
||||
env = "OPENSHELL_K8S_SANDBOX_RUNTIME_NETWORK_POLICY_ENFORCED",
|
||||
default_value_t = false
|
||||
)]
|
||||
proxy_pod_network_policy_enforced: bool,
|
||||
sandbox_runtime_network_policy_enforced: bool,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_K8S_PROXY_POD_BOUNDARY_PORT",
|
||||
env = "OPENSHELL_K8S_SANDBOX_RUNTIME_BOUNDARY_PORT",
|
||||
default_value_t = 5500
|
||||
)]
|
||||
proxy_pod_boundary_port: u16,
|
||||
sandbox_runtime_boundary_port: u16,
|
||||
|
||||
/// Corporate HTTP forward proxy for policy-approved TLS CONNECT egress.
|
||||
#[arg(long, env = "OPENSHELL_UPSTREAM_PROXY")]
|
||||
@@ -255,9 +255,9 @@ async fn main() -> Result<()> {
|
||||
.supervisor_image
|
||||
.unwrap_or_else(openshell_core::config::default_supervisor_image),
|
||||
supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(),
|
||||
proxy_pod: KubernetesProxyPodConfig {
|
||||
network_policy_enforced: args.proxy_pod_network_policy_enforced,
|
||||
boundary_port: args.proxy_pod_boundary_port,
|
||||
sandbox_runtime: KubernetesSandboxRuntimeConfig {
|
||||
network_policy_enforced: args.sandbox_runtime_network_policy_enforced,
|
||||
boundary_port: args.sandbox_runtime_boundary_port,
|
||||
},
|
||||
https_proxy: args.https_proxy,
|
||||
no_proxy: args.no_proxy,
|
||||
|
||||
+25
-27
@@ -1,7 +1,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Kubernetes resources owned by the RFC 0012 proxy-pod topology.
|
||||
//! Kubernetes resources that place and protect the `OpenShell` sandbox runtime.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
@@ -15,14 +15,14 @@ use kube::core::ObjectMeta;
|
||||
use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose};
|
||||
|
||||
use crate::isolation::{
|
||||
BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesProxyPodNetworkFence,
|
||||
KubernetesProxyPodNetworkFenceSpec,
|
||||
BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesSandboxRuntimeNetworkFence,
|
||||
KubernetesSandboxRuntimeNetworkFenceSpec,
|
||||
};
|
||||
|
||||
pub const SANDBOX_SECRET_COMPONENT: &str = "sandbox-bootstrap";
|
||||
pub const SUPERVISOR_SECRET_COMPONENT: &str = "supervisor-bootstrap";
|
||||
pub const BOUNDARY_CONFIG_KEY: &str = "boundary.json";
|
||||
pub const TOPOLOGY_PAYLOAD_KEY: &str = "topology.json";
|
||||
pub const BACKEND_DESCRIPTOR_KEY: &str = "runtime-descriptor.json";
|
||||
pub const BOUNDARY_CERTIFICATE_KEY: &str = "tls.crt";
|
||||
pub const BOUNDARY_PRIVATE_KEY: &str = "tls.key";
|
||||
pub const SUPERVISOR_AUTH_BUNDLE_KEY: &str = "auth.json";
|
||||
@@ -32,7 +32,7 @@ pub const SANDBOX_BOOTSTRAP_INPUT_PATH: &str = "/.openshell/bootstrap-input";
|
||||
pub const BOUNDARY_CONFIG_PATH: &str = "/.openshell/state/bootstrap/boundary.json";
|
||||
pub const BOUNDARY_CERTIFICATE_PATH: &str = "/.openshell/state/bootstrap/tls.crt";
|
||||
pub const BOUNDARY_PRIVATE_KEY_PATH: &str = "/.openshell/state/bootstrap/tls.key";
|
||||
pub const TOPOLOGY_PAYLOAD_PATH: &str = "/.openshell/supervisor/topology.json";
|
||||
pub const BACKEND_DESCRIPTOR_PATH: &str = "/.openshell/supervisor/runtime-descriptor.json";
|
||||
pub const SUPERVISOR_AUTH_BUNDLE_PATH: &str = "/.openshell/supervisor/auth.json";
|
||||
pub const PROXY_CA_CERTIFICATE_PATH: &str = "/.openshell/supervisor/proxy-ca.crt";
|
||||
pub const PROXY_CA_PRIVATE_KEY_PATH: &str = "/.openshell/supervisor/proxy-ca.key";
|
||||
@@ -64,7 +64,7 @@ pub fn generate_proxy_ca_material() -> Result<ProxyCaMaterial, String> {
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ProxyPodNames {
|
||||
pub struct SandboxRuntimeNames {
|
||||
pub sandbox_secret: String,
|
||||
pub supervisor_secret: String,
|
||||
pub boundary_service: String,
|
||||
@@ -73,7 +73,7 @@ pub struct ProxyPodNames {
|
||||
pub control_policy: String,
|
||||
}
|
||||
|
||||
impl ProxyPodNames {
|
||||
impl SandboxRuntimeNames {
|
||||
#[must_use]
|
||||
pub fn new(sandbox_id: &str) -> Self {
|
||||
let suffix = sandbox_id.to_ascii_lowercase();
|
||||
@@ -112,11 +112,11 @@ pub fn pair_label_value(sandbox_id: &str) -> String {
|
||||
#[must_use]
|
||||
pub fn workload_fence(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
names: &SandboxRuntimeNames,
|
||||
sandbox_id: &str,
|
||||
boundary_port: u16,
|
||||
) -> KubernetesProxyPodNetworkFence {
|
||||
KubernetesProxyPodNetworkFenceSpec {
|
||||
) -> KubernetesSandboxRuntimeNetworkFence {
|
||||
KubernetesSandboxRuntimeNetworkFenceSpec {
|
||||
namespace: namespace.to_string(),
|
||||
policy_name: names.workload_policy.clone(),
|
||||
pair_label_value: pair_label_value(sandbox_id),
|
||||
@@ -128,7 +128,7 @@ pub fn workload_fence(
|
||||
#[must_use]
|
||||
pub fn boundary_service(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
names: &SandboxRuntimeNames,
|
||||
sandbox_id: &str,
|
||||
boundary_port: u16,
|
||||
owner: OwnerReference,
|
||||
@@ -154,7 +154,7 @@ pub fn boundary_service(
|
||||
#[must_use]
|
||||
pub fn control_deployment(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
names: &SandboxRuntimeNames,
|
||||
sandbox_id: &str,
|
||||
sandbox_name: &str,
|
||||
gateway_id: &str,
|
||||
@@ -241,10 +241,8 @@ pub fn control_deployment(
|
||||
}
|
||||
let mut command = vec![
|
||||
"/openshell-supervisor".to_string(),
|
||||
"--topology-backend-name".to_string(),
|
||||
crate::isolation::BACKEND_NAME.to_string(),
|
||||
"--topology-payload-file".to_string(),
|
||||
TOPOLOGY_PAYLOAD_PATH.to_string(),
|
||||
"--backend-descriptor-file".to_string(),
|
||||
BACKEND_DESCRIPTOR_PATH.to_string(),
|
||||
"--auth-bundle-file".to_string(),
|
||||
SUPERVISOR_AUTH_BUNDLE_PATH.to_string(),
|
||||
"--workdir".to_string(),
|
||||
@@ -361,7 +359,7 @@ pub fn control_deployment(
|
||||
#[must_use]
|
||||
pub fn control_egress_policy(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
names: &SandboxRuntimeNames,
|
||||
sandbox_id: &str,
|
||||
owner: OwnerReference,
|
||||
) -> NetworkPolicy {
|
||||
@@ -391,7 +389,7 @@ pub fn control_egress_policy(
|
||||
#[must_use]
|
||||
pub fn sandbox_bootstrap_secret(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
names: &SandboxRuntimeNames,
|
||||
sandbox_id: &str,
|
||||
boundary_config: Vec<u8>,
|
||||
boundary_certificate: Vec<u8>,
|
||||
@@ -427,9 +425,9 @@ pub fn sandbox_bootstrap_secret(
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn supervisor_bootstrap_secret(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
names: &SandboxRuntimeNames,
|
||||
sandbox_id: &str,
|
||||
topology_payload: Vec<u8>,
|
||||
backend_descriptor: Vec<u8>,
|
||||
supervisor_auth_bundle: Vec<u8>,
|
||||
proxy_ca_certificate: Vec<u8>,
|
||||
proxy_ca_private_key: Vec<u8>,
|
||||
@@ -445,8 +443,8 @@ pub fn supervisor_bootstrap_secret(
|
||||
},
|
||||
data: Some(BTreeMap::from([
|
||||
(
|
||||
TOPOLOGY_PAYLOAD_KEY.to_string(),
|
||||
ByteString(topology_payload),
|
||||
BACKEND_DESCRIPTOR_KEY.to_string(),
|
||||
ByteString(backend_descriptor),
|
||||
),
|
||||
(
|
||||
SUPERVISOR_AUTH_BUNDLE_KEY.to_string(),
|
||||
@@ -536,7 +534,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn service_selects_only_the_workload_boundary() {
|
||||
let names = ProxyPodNames::new("4b67c0d0-1111-2222-3333-444444444444");
|
||||
let names = SandboxRuntimeNames::new("4b67c0d0-1111-2222-3333-444444444444");
|
||||
let service = boundary_service("sandbox", &names, "pair", 5500, owner());
|
||||
assert_eq!(
|
||||
service.spec.unwrap().selector.unwrap()[BOUNDARY_ROLE_LABEL],
|
||||
@@ -551,7 +549,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn control_policy_explicitly_allows_control_egress() {
|
||||
let names = ProxyPodNames::new("pair");
|
||||
let names = SandboxRuntimeNames::new("pair");
|
||||
let policy = control_egress_policy("sandbox", &names, "pair", owner());
|
||||
let spec = policy.spec.unwrap();
|
||||
assert_eq!(spec.policy_types.unwrap(), ["Egress"]);
|
||||
@@ -560,7 +558,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn control_deployment_is_singleton_ready_and_unprivileged() {
|
||||
let names = ProxyPodNames::new("pair");
|
||||
let names = SandboxRuntimeNames::new("pair");
|
||||
let deployment = control_deployment(
|
||||
"sandbox",
|
||||
&names,
|
||||
@@ -665,7 +663,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn bootstrap_secrets_are_immutable_and_split_by_trust_domain() {
|
||||
let names = ProxyPodNames::new("pair");
|
||||
let names = SandboxRuntimeNames::new("pair");
|
||||
let sandbox = sandbox_bootstrap_secret(
|
||||
"sandbox",
|
||||
&names,
|
||||
@@ -720,7 +718,7 @@ mod tests {
|
||||
PROXY_CA_CERTIFICATE_KEY.to_string(),
|
||||
PROXY_CA_PRIVATE_KEY.to_string(),
|
||||
SUPERVISOR_AUTH_BUNDLE_KEY.to_string(),
|
||||
TOPOLOGY_PAYLOAD_KEY.to_string(),
|
||||
BACKEND_DESCRIPTOR_KEY.to_string(),
|
||||
])
|
||||
);
|
||||
}
|
||||
@@ -299,8 +299,8 @@ discovery endpoint or its TLS CA.
|
||||
| supervisor.image.pullPolicy | string | `nil` | Sandbox supervisor pull policy. Leave unset to use the Kubernetes image default. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. |
|
||||
| supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. |
|
||||
| supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. |
|
||||
| supervisor.proxyPod.boundaryPort | int | `5500` | Workload boundary TLS listener port. |
|
||||
| supervisor.proxyPod.networkPolicyEnforced | bool | `false` | Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. |
|
||||
| supervisor.sandboxRuntime.boundaryPort | int | `5500` | Workload boundary TLS listener port. |
|
||||
| supervisor.sandboxRuntime.networkPolicyEnforced | bool | `false` | Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. |
|
||||
| tolerations | list | `[]` | Tolerations for the gateway pod. |
|
||||
| upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. |
|
||||
| upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. |
|
||||
|
||||
+1
-1
@@ -2,5 +2,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
supervisor:
|
||||
proxyPod:
|
||||
sandboxRuntime:
|
||||
networkPolicyEnforced: true
|
||||
@@ -218,9 +218,9 @@ data:
|
||||
gateway_namespace = {{ .Release.Namespace | quote }}
|
||||
gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} }
|
||||
|
||||
[openshell.drivers.kubernetes.proxy_pod]
|
||||
network_policy_enforced = {{ .Values.supervisor.proxyPod.networkPolicyEnforced }}
|
||||
boundary_port = {{ .Values.supervisor.proxyPod.boundaryPort | default 5500 }}
|
||||
[openshell.drivers.kubernetes.sandbox_runtime]
|
||||
network_policy_enforced = {{ .Values.supervisor.sandboxRuntime.networkPolicyEnforced }}
|
||||
boundary_port = {{ .Values.supervisor.sandboxRuntime.boundaryPort | default 5500 }}
|
||||
|
||||
{{- if not $credentialDrivers }}
|
||||
|
||||
|
||||
@@ -26,7 +26,7 @@ tests:
|
||||
resources: ["networkpolicies"]
|
||||
verbs: ["get", "create", "patch", "update"]
|
||||
|
||||
- it: preserves proxy-pod fence permissions when gateway isolation is disabled
|
||||
- it: preserves sandbox-runtime fence permissions when gateway isolation is disabled
|
||||
set:
|
||||
server.drivers.kubernetes.workspaceMode: managed
|
||||
networkPolicy.enabled: false
|
||||
@@ -141,7 +141,7 @@ tests:
|
||||
resources: ["secrets"]
|
||||
any: true
|
||||
|
||||
- it: grants managed proxy-pod companion permissions
|
||||
- it: grants managed sandbox-runtime companion permissions
|
||||
set:
|
||||
server.drivers.kubernetes.workspaceMode: managed
|
||||
asserts:
|
||||
@@ -158,7 +158,7 @@ tests:
|
||||
resources: ["secrets"]
|
||||
verbs: ["create"]
|
||||
|
||||
- it: grants operator proxy-pod companion permissions
|
||||
- it: grants operator sandbox-runtime companion permissions
|
||||
set:
|
||||
server.drivers.kubernetes.workspaceMode: operator
|
||||
asserts:
|
||||
|
||||
@@ -47,7 +47,7 @@ supervisor:
|
||||
pullPolicy: null
|
||||
# -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed.
|
||||
tag: ""
|
||||
proxyPod:
|
||||
sandboxRuntime:
|
||||
# -- Required operator acknowledgement that the cluster CNI enforces NetworkPolicy.
|
||||
networkPolicyEnforced: false
|
||||
# -- Workload boundary TLS listener port.
|
||||
|
||||
@@ -36,7 +36,7 @@ oc create ns openshell
|
||||
helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \
|
||||
--version <version> \
|
||||
--namespace openshell \
|
||||
--set supervisor.proxyPod.networkPolicyEnforced=true
|
||||
--set supervisor.sandboxRuntime.networkPolicyEnforced=true
|
||||
```
|
||||
|
||||
The driver reads the namespace's `openshift.io/sa.scc.uid-range` annotation and
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
---
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
title: "Kubernetes Sandbox Topology"
|
||||
sidebar-title: "Topology"
|
||||
description: "Understand the capability-free sandbox and supervisor topology used for Kubernetes workloads."
|
||||
title: "Kubernetes Sandbox Runtime"
|
||||
sidebar-title: "Sandbox runtime"
|
||||
description: "Understand how Kubernetes places and protects the sandbox runtime and supervisor."
|
||||
keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Network Policy, Seccomp, Landlock"
|
||||
position: 2
|
||||
---
|
||||
@@ -14,7 +14,7 @@ policy decisions, and upstream connections.
|
||||
|
||||
## Understand the Components
|
||||
|
||||
The Kubernetes driver always uses this topology:
|
||||
The Kubernetes driver always uses this placement:
|
||||
|
||||
```mermaid
|
||||
flowchart LR
|
||||
@@ -76,7 +76,7 @@ policy-approved upstream destinations.
|
||||
Kubernetes policies are additive. Keep sandbox namespaces under administrative
|
||||
control so another principal cannot add permissive policies, create Pods with
|
||||
OpenShell labels, or read bootstrap Secrets. Set
|
||||
`supervisor.proxyPod.networkPolicyEnforced: true` only after you verify that the
|
||||
`supervisor.sandboxRuntime.networkPolicyEnforced: true` only after you verify that the
|
||||
cluster CNI enforces both ingress and egress policies for these namespaces.
|
||||
|
||||
## Bootstrap a Sandbox
|
||||
@@ -105,7 +105,7 @@ egress fence until Kubernetes confirms that the workload Pod is gone.
|
||||
|
||||
## Check Cluster Requirements
|
||||
|
||||
The topology requires the following cluster behavior:
|
||||
This architecture requires the following cluster behavior:
|
||||
|
||||
- Linux nodes and a container runtime that permits an unprivileged process to
|
||||
install a nested seccomp user-notification filter under `RuntimeDefault`.
|
||||
@@ -203,8 +203,8 @@ The most commonly changed values are:
|
||||
| `server.auth.allowUnauthenticatedUsers` | Accept user-facing calls without OIDC or mTLS credentials. Use only for trusted local development or a fully trusted access proxy. |
|
||||
| `server.enableLoopbackServiceHttp` | Enable local plaintext HTTP for loopback sandbox service URLs. Defaults to `true`. |
|
||||
| `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. |
|
||||
| `supervisor.proxyPod.networkPolicyEnforced` | Required acknowledgement that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. |
|
||||
| `supervisor.proxyPod.boundaryPort` | Non-privileged TLS port used between paired supervisor and sandbox Pods. |
|
||||
| `supervisor.sandboxRuntime.networkPolicyEnforced` | Required acknowledgement that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. |
|
||||
| `supervisor.sandboxRuntime.boundaryPort` | Non-privileged TLS port used between paired supervisor and sandbox Pods. |
|
||||
| `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). |
|
||||
|
||||
Use a values file for repeatable deployments:
|
||||
@@ -311,7 +311,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency
|
||||
|
||||
## Next Steps
|
||||
|
||||
- Kubernetes sandboxes use a separate sandbox Pod and supervisor Deployment; refer to [Topology](/kubernetes/topology).
|
||||
- Kubernetes sandboxes use a separate sandbox Pod and supervisor Deployment; refer to [Sandbox runtime](/kubernetes/sandbox-runtime).
|
||||
- To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates).
|
||||
- To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress).
|
||||
- To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control).
|
||||
|
||||
@@ -94,7 +94,7 @@ future version. To migrate an existing file:
|
||||
omit Docker or Podman `sandbox_pids_limit` to use OpenShell's default limit
|
||||
of 2048, and omit Podman `health_check_interval_secs` to disable health
|
||||
checks. Explicit zero values are invalid.
|
||||
7. Remove local Docker, Podman, or VM `grpc_endpoint` when the topology-derived
|
||||
7. Remove local Docker, Podman, or VM `grpc_endpoint` when the driver-derived
|
||||
callback is correct, or retain it as an explicit override. Kubernetes raw
|
||||
TOML requires an explicit endpoint; Helm derives one from the release's
|
||||
gateway Service. New VM root filesystems use an image-provided `sandbox`
|
||||
@@ -546,7 +546,7 @@ image_pull_secrets = ["regcred"]
|
||||
# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:<version>"
|
||||
supervisor_image_pull_policy = "if_not_present"
|
||||
|
||||
[openshell.drivers.kubernetes.proxy_pod]
|
||||
[openshell.drivers.kubernetes.sandbox_runtime]
|
||||
# Set this only after verifying that the cluster CNI enforces both ingress and
|
||||
# egress NetworkPolicy in every sandbox namespace.
|
||||
network_policy_enforced = true
|
||||
@@ -619,7 +619,7 @@ enabled = true
|
||||
gateway_namespace = "openshell"
|
||||
gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" }
|
||||
|
||||
[openshell.drivers.kubernetes.proxy_pod]
|
||||
[openshell.drivers.kubernetes.sandbox_runtime]
|
||||
# Required acknowledgement that the cluster CNI enforces NetworkPolicy and the
|
||||
# sandbox namespaces prevent untrusted policy, pod, label, and Secret changes.
|
||||
network_policy_enforced = true
|
||||
@@ -699,7 +699,7 @@ image_pull_policy = "if_not_present"
|
||||
# Value assigned to the openshell.sandbox_namespace label on sandbox containers.
|
||||
sandbox_label = "docker-dev"
|
||||
# Optional override. When omitted, the gateway derives
|
||||
# https://host.openshell.internal:<gateway-port> for this topology.
|
||||
# https://host.openshell.internal:<gateway-port> for this driver.
|
||||
grpc_endpoint = "https://host.openshell.internal:17670"
|
||||
# Contains both /openshell-sandbox and /openshell-supervisor. Defaults to the
|
||||
# gateway version; override to pin a specific build.
|
||||
@@ -905,7 +905,7 @@ state_dir = "/var/lib/openshell/vm"
|
||||
driver_dir = "/usr/local/libexec/openshell"
|
||||
default_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest"
|
||||
# Optional override. When omitted, the gateway derives
|
||||
# https://host.openshell.internal:<gateway-port> for the VM topology.
|
||||
# https://host.openshell.internal:<gateway-port> for the VM driver.
|
||||
grpc_endpoint = "https://host.openshell.internal:17670"
|
||||
# Empty falls back to default_image.
|
||||
bootstrap_image = "ghcr.io/nvidia/openshell-community/sandboxes/base:latest"
|
||||
|
||||
@@ -423,8 +423,8 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA
|
||||
| `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. |
|
||||
| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the trusted runtime image that provides the `openshell-sandbox` and `openshell-supervisor` binaries. The default repository with an empty tag uses the version pinned into the gateway. |
|
||||
| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. |
|
||||
| `proxy_pod.network_policy_enforced` | `supervisor.proxyPod.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. |
|
||||
| `proxy_pod.boundary_port` | `supervisor.proxyPod.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. |
|
||||
| `sandbox_runtime.network_policy_enforced` | `supervisor.sandboxRuntime.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. |
|
||||
| `sandbox_runtime.boundary_port` | `supervisor.sandboxRuntime.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. |
|
||||
| `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. |
|
||||
| `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. |
|
||||
| `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. The Secret mounts only in the supervisor Pod. |
|
||||
@@ -442,7 +442,8 @@ remain limited to the explicitly configured TLS and image-pull Secret names.
|
||||
The driver creates copies only in gateway-owned managed namespaces. Do not
|
||||
reuse the gateway ServiceAccount for unrelated workloads.
|
||||
|
||||
The Kubernetes driver always uses a capability-free proxy-pod topology. The
|
||||
The Kubernetes driver always places the sandbox runtime in the workload Pod and
|
||||
the supervisor in a separate Deployment. The
|
||||
workload Pod runs `openshell-sandbox` as the same non-root UID/GID as the agent
|
||||
and requests no added Linux capabilities. A separate Deployment runs
|
||||
`openshell-supervisor`, authenticates to the gateway with a sandbox JWT, and
|
||||
|
||||
@@ -22,7 +22,7 @@ ENV OPENSHELL_COMPUTE_DRIVER=kubernetes \
|
||||
OPENSHELL_SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE} \
|
||||
OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=if_not_present \
|
||||
OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \
|
||||
OPENSHELL_K8S_PROXY_POD_NETWORK_POLICY_ENFORCED=true
|
||||
OPENSHELL_K8S_SANDBOX_RUNTIME_NETWORK_POLICY_ENFORCED=true
|
||||
|
||||
USER 1000:1000
|
||||
EXPOSE 8080
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
//! Kubernetes wiring coverage for authenticated corporate forward proxies.
|
||||
//!
|
||||
//! The shell wrapper configures the gateway before it starts, creates the
|
||||
//! credential Secret, and enables proxy-pod topology. This test starts a proxy
|
||||
//! credential Secret, and enables the Kubernetes sandbox runtime. This test starts a proxy
|
||||
//! and an HTTPS upstream on the host visible to sandbox pods, then proves that
|
||||
//! a permitted request uses authenticated CONNECT while a policy-denied port
|
||||
//! never reaches the proxy.
|
||||
|
||||
@@ -263,10 +263,10 @@ cleanup() {
|
||||
echo "=== gateway sandbox records ==="
|
||||
"${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" \
|
||||
sandbox list --all-workspaces --output json 2>&1 || true
|
||||
echo "=== proxy-pod supervisor Deployments ==="
|
||||
echo "=== sandbox-runtime supervisor Deployments ==="
|
||||
kctl -n "${NAMESPACE}" get deployments \
|
||||
-l "openshell.ai/boundary-role=supervisor" -o yaml 2>&1 || true
|
||||
echo "=== proxy-pod supervisor logs (last 200 lines each) ==="
|
||||
echo "=== sandbox-runtime supervisor logs (last 200 lines each) ==="
|
||||
while IFS= read -r supervisor_pod; do
|
||||
[ -n "${supervisor_pod}" ] || continue
|
||||
echo "--- ${supervisor_pod} ---"
|
||||
@@ -284,7 +284,7 @@ cleanup() {
|
||||
kctl -n "${NAMESPACE}" logs "$(kube_workload_ref "${RELEASE_NAME}")" \
|
||||
--since=20m \
|
||||
--all-containers --prefix 2>&1 \
|
||||
| grep -Ei "sandbox phase changed|start_sandbox|stop_sandbox|supervisor session|proxy-pod|bootstrap" \
|
||||
| grep -Ei "sandbox phase changed|start_sandbox|stop_sandbox|supervisor session|sandbox-runtime|bootstrap" \
|
||||
|| true
|
||||
echo "=== gateway logs (last 200 lines) ==="
|
||||
kctl -n "${NAMESPACE}" logs \
|
||||
|
||||
@@ -550,7 +550,8 @@ kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\
|
||||
kubectl -n <sandbox-namespace> get sandbox <sandbox-name> -o jsonpath='{.spec.template.spec.serviceAccountName}{"\n"}'
|
||||
```
|
||||
|
||||
Kubernetes always uses the proxy-pod topology. Helm must render
|
||||
The Kubernetes driver creates a sandbox workload Pod and a separate supervisor
|
||||
Deployment. Helm must render
|
||||
`network_policy_enforced = true`. This is an explicit operator assertion that
|
||||
the cluster CNI enforces Kubernetes NetworkPolicy; the Kubernetes API cannot
|
||||
attest enforcement. Run sandboxes only in a trusted namespace
|
||||
@@ -569,7 +570,7 @@ the sandbox over per-sandbox TLS with server-certificate verification plus
|
||||
bootstrap-token client authentication, and owns gateway policy, provider
|
||||
credentials, DNS, and mediated upstream connections.
|
||||
|
||||
Inspect the complete companion set when a proxy-pod sandbox remains Starting
|
||||
Inspect all driver-managed resources when a Kubernetes sandbox remains Starting
|
||||
or loses readiness:
|
||||
|
||||
```bash
|
||||
|
||||
+5
-5
@@ -190,9 +190,9 @@ run = [
|
||||
"OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh",
|
||||
]
|
||||
|
||||
["e2e:kubernetes:proxy-pod"]
|
||||
description = "Run Kubernetes e2e with the sandbox/supervisor proxy-pod topology"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml" }
|
||||
["e2e:kubernetes:isolation"]
|
||||
description = "Run Kubernetes e2e with the workload network fence and separate supervisor"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sandbox-runtime.yaml" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
@@ -210,13 +210,13 @@ run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debu
|
||||
|
||||
["e2e:kubernetes:workspace-managed"]
|
||||
description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml:deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" }
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sandbox-runtime.yaml:deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:kubernetes:workspace-operator"]
|
||||
description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml:deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" }
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sandbox-runtime.yaml:deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user