mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-30 07:33:59 +08:00
refactor(windows): separate platform crate roots
Signed-off-by: Akber Raza <akberr@nvidia.com>
This commit is contained in:
Akber Raza
@@ -10,6 +10,9 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
path = "src/lib_entry.rs"
|
||||
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
serde = { workspace = true }
|
||||
|
||||
@@ -1,11 +1,9 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
//! Docker compute driver.
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub use windows::{DockerComputeConfig, default_docker_supervisor_image};
|
||||
#![allow(clippy::result_large_err)]
|
||||
|
||||
use bollard::Docker;
|
||||
use bollard::errors::Error as BollardError;
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("lib.rs");
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
include!("lib_win.rs");
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
mod windows;
|
||||
|
||||
pub use windows::{DockerComputeConfig, default_docker_supervisor_image};
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
[package]
|
||||
name = "openshell-driver-kubernetes"
|
||||
autobins = false
|
||||
description = "Kubernetes compute driver for OpenShell"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
@@ -12,7 +13,7 @@ repository.workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "openshell-driver-kubernetes"
|
||||
path = "src/main.rs"
|
||||
path = "src/main_entry.rs"
|
||||
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("main_unix.rs");
|
||||
use clap::{ArgAction, Parser};
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use std::net::SocketAddr;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use openshell_core::VERSION;
|
||||
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("main.rs");
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
include!("main_win.rs");
|
||||
@@ -1,181 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use clap::{ArgAction, Parser};
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use std::net::SocketAddr;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use openshell_core::VERSION;
|
||||
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
|
||||
use openshell_driver_kubernetes::{
|
||||
AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME,
|
||||
KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig,
|
||||
SupervisorSideloadMethod, SupervisorTopology,
|
||||
};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "openshell-driver-kubernetes")]
|
||||
#[command(version = VERSION)]
|
||||
struct Args {
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_COMPUTE_DRIVER_BIND",
|
||||
default_value = "127.0.0.1:50061"
|
||||
)]
|
||||
bind_address: SocketAddr,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")]
|
||||
log_level: String,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")]
|
||||
sandbox_namespace: String,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT",
|
||||
default_value = DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME
|
||||
)]
|
||||
sandbox_service_account: String,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")]
|
||||
sandbox_image: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")]
|
||||
sandbox_image_pull_policy: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SANDBOX_IMAGE_PULL_SECRETS",
|
||||
value_delimiter = ','
|
||||
)]
|
||||
sandbox_image_pull_secrets: Vec<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
|
||||
grpc_endpoint: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH",
|
||||
default_value = "/run/openshell/ssh.sock"
|
||||
)]
|
||||
sandbox_ssh_socket_path: String,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_CLIENT_TLS_SECRET_NAME")]
|
||||
client_tls_secret_name: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")]
|
||||
host_gateway_ip: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")]
|
||||
supervisor_image: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")]
|
||||
supervisor_image_pull_policy: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SUPERVISOR_SIDELOAD_METHOD",
|
||||
default_value = "image-volume"
|
||||
)]
|
||||
supervisor_sideload_method: SupervisorSideloadMethod,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")]
|
||||
topology: SupervisorTopology,
|
||||
|
||||
#[arg(
|
||||
long = "sidecar-proxy-uid",
|
||||
alias = "proxy-uid",
|
||||
env = "OPENSHELL_K8S_SIDECAR_PROXY_UID",
|
||||
default_value_t = DEFAULT_PROXY_UID
|
||||
)]
|
||||
sidecar_proxy_uid: u32,
|
||||
|
||||
#[arg(
|
||||
long = "sidecar-process-binary-aware-network-policy",
|
||||
env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY",
|
||||
default_value_t = true,
|
||||
action = ArgAction::Set
|
||||
)]
|
||||
sidecar_process_binary_aware_network_policy: bool,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")]
|
||||
enable_user_namespaces: bool,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_K8S_APP_ARMOR_PROFILE")]
|
||||
app_armor_profile: Option<AppArmorProfile>,
|
||||
|
||||
/// Lifetime (seconds) of the projected `ServiceAccount` token
|
||||
/// kubelet writes into each sandbox pod for the `IssueSandboxToken`
|
||||
/// bootstrap exchange. Kubelet enforces a minimum of 600s; the
|
||||
/// gateway clamps values outside `[600, 86400]`. Default 3600.
|
||||
#[arg(long, env = "OPENSHELL_K8S_SA_TOKEN_TTL_SECS", default_value_t = 3600)]
|
||||
sa_token_ttl_secs: i64,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")]
|
||||
provider_spiffe_workload_api_socket_path: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_K8S_SANDBOX_UID")]
|
||||
sandbox_uid: Option<u32>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_K8S_SANDBOX_GID")]
|
||||
sandbox_gid: Option<u32>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)),
|
||||
)
|
||||
.init();
|
||||
|
||||
let driver = KubernetesComputeDriver::new(KubernetesComputeConfig {
|
||||
namespace: args.sandbox_namespace,
|
||||
service_account_name: args.sandbox_service_account,
|
||||
default_image: args.sandbox_image.unwrap_or_default(),
|
||||
image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(),
|
||||
image_pull_secrets: args.sandbox_image_pull_secrets,
|
||||
supervisor_image: args
|
||||
.supervisor_image
|
||||
.unwrap_or_else(openshell_core::config::default_supervisor_image),
|
||||
supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(),
|
||||
supervisor_sideload_method: args.supervisor_sideload_method,
|
||||
topology: args.topology,
|
||||
sidecar: KubernetesSidecarConfig {
|
||||
proxy_uid: args.sidecar_proxy_uid,
|
||||
process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy,
|
||||
},
|
||||
grpc_endpoint: args.grpc_endpoint.unwrap_or_default(),
|
||||
ssh_socket_path: args.sandbox_ssh_socket_path,
|
||||
client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(),
|
||||
host_gateway_ip: args.host_gateway_ip.unwrap_or_default(),
|
||||
enable_user_namespaces: args.enable_user_namespaces,
|
||||
app_armor_profile: args.app_armor_profile,
|
||||
workspace_default_storage_size: std::env::var(
|
||||
"OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE",
|
||||
)
|
||||
.unwrap_or_else(|_| {
|
||||
openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string()
|
||||
}),
|
||||
default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME")
|
||||
.unwrap_or_default(),
|
||||
sa_token_ttl_secs: args.sa_token_ttl_secs,
|
||||
provider_spiffe_workload_api_socket_path: args
|
||||
.provider_spiffe_workload_api_socket_path
|
||||
.unwrap_or_default(),
|
||||
sandbox_uid: args.sandbox_uid,
|
||||
sandbox_gid: args.sandbox_gid,
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()?;
|
||||
|
||||
info!(address = %args.bind_address, "Starting Kubernetes compute driver");
|
||||
tonic::transport::Server::builder()
|
||||
.add_service(ComputeDriverServer::new(ComputeDriverService::new(driver)))
|
||||
.serve(args.bind_address)
|
||||
.await
|
||||
.into_diagnostic()
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
fn main() {
|
||||
eprintln!("openshell-driver-kubernetes is unsupported on Windows");
|
||||
std::process::exit(1);
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
[package]
|
||||
name = "openshell-driver-podman"
|
||||
autobins = false
|
||||
description = "Podman compute driver for OpenShell"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
@@ -10,9 +11,12 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
path = "src/lib_entry.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "openshell-driver-podman"
|
||||
path = "src/main.rs"
|
||||
path = "src/main_entry.rs"
|
||||
|
||||
[target.'cfg(target_os = "windows")'.dependencies]
|
||||
serde = { workspace = true }
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
mod windows;
|
||||
pub(crate) mod client;
|
||||
pub mod config;
|
||||
pub(crate) mod container;
|
||||
pub mod driver;
|
||||
pub mod grpc;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_utils;
|
||||
pub(crate) mod watcher;
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub use windows::PodmanComputeConfig;
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("lib_unix.rs");
|
||||
pub use config::PodmanComputeConfig;
|
||||
pub use driver::PodmanComputeDriver;
|
||||
pub use grpc::ComputeDriverService;
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("lib.rs");
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
include!("lib_win.rs");
|
||||
@@ -1,15 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
pub(crate) mod client;
|
||||
pub mod config;
|
||||
pub(crate) mod container;
|
||||
pub mod driver;
|
||||
pub mod grpc;
|
||||
#[cfg(test)]
|
||||
pub(crate) mod test_utils;
|
||||
pub(crate) mod watcher;
|
||||
|
||||
pub use config::PodmanComputeConfig;
|
||||
pub use driver::PodmanComputeDriver;
|
||||
pub use grpc::ComputeDriverService;
|
||||
@@ -0,0 +1,6 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
mod windows;
|
||||
|
||||
pub use windows::PodmanComputeConfig;
|
||||
@@ -1,11 +1,185 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn main() {
|
||||
eprintln!("openshell-driver-podman is unsupported on Windows");
|
||||
std::process::exit(1);
|
||||
use clap::Parser;
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use openshell_core::VERSION;
|
||||
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
|
||||
use openshell_driver_podman::config::{
|
||||
DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT,
|
||||
ImagePullPolicy,
|
||||
};
|
||||
use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "openshell-driver-podman")]
|
||||
#[command(version = VERSION)]
|
||||
struct Args {
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_COMPUTE_DRIVER_BIND",
|
||||
default_value = "127.0.0.1:50061"
|
||||
)]
|
||||
bind_address: SocketAddr,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")]
|
||||
log_level: String,
|
||||
|
||||
/// Path to the Podman API Unix socket.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_SOCKET")]
|
||||
podman_socket: Option<PathBuf>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")]
|
||||
sandbox_image: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY",
|
||||
default_value_t = ImagePullPolicy::Missing
|
||||
)]
|
||||
sandbox_image_pull_policy: ImagePullPolicy,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
|
||||
grpc_endpoint: Option<String>,
|
||||
|
||||
/// Port the gateway server is listening on.
|
||||
///
|
||||
/// Used when `--grpc-endpoint` is not set to auto-detect the endpoint
|
||||
/// that sandbox containers dial back to.
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_GATEWAY_PORT",
|
||||
default_value_t = openshell_core::config::DEFAULT_SERVER_PORT
|
||||
)]
|
||||
gateway_port: u16,
|
||||
|
||||
/// Host gateway IP used for sandbox host aliases.
|
||||
///
|
||||
/// Empty uses Podman's `host-gateway` resolver.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_HOST_GATEWAY_IP")]
|
||||
host_gateway_ip: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH",
|
||||
default_value = openshell_core::container_paths::SSH_SOCKET_PATH
|
||||
)]
|
||||
sandbox_ssh_socket_path: String,
|
||||
|
||||
/// Podman bridge network name.
|
||||
#[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)]
|
||||
network_name: String,
|
||||
|
||||
/// Container stop timeout in seconds (SIGTERM → SIGKILL).
|
||||
#[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)]
|
||||
stop_timeout: u32,
|
||||
|
||||
/// Container cgroup PID limit for sandbox containers. Set 0 to inherit
|
||||
/// Podman's runtime/default PID limit.
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SANDBOX_PIDS_LIMIT",
|
||||
default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT
|
||||
)]
|
||||
sandbox_pids_limit: i64,
|
||||
|
||||
/// OCI image containing the openshell-sandbox supervisor binary.
|
||||
#[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")]
|
||||
supervisor_image: Option<String>,
|
||||
|
||||
/// Host path to the CA certificate for sandbox mTLS.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_TLS_CA")]
|
||||
podman_tls_ca: Option<PathBuf>,
|
||||
|
||||
/// Host path to the client certificate for sandbox mTLS.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_TLS_CERT")]
|
||||
podman_tls_cert: Option<PathBuf>,
|
||||
|
||||
/// Host path to the client private key for sandbox mTLS.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")]
|
||||
podman_tls_key: Option<PathBuf>,
|
||||
|
||||
/// Corporate forward proxy URL for the supervisor's upstream TLS dials,
|
||||
/// in explicit `http://host:port` form (scheme and port required).
|
||||
/// Credentials must not be embedded in the URL; use
|
||||
/// `--sandbox-proxy-auth-file` instead.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_HTTPS_PROXY")]
|
||||
sandbox_https_proxy: Option<String>,
|
||||
|
||||
/// Comma-separated `NO_PROXY` list injected alongside the proxy URL.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_NO_PROXY")]
|
||||
sandbox_no_proxy: Option<String>,
|
||||
|
||||
/// Path to a file containing the corporate proxy credentials as
|
||||
/// `user:pass`. Delivered to the supervisor through a root-only secret
|
||||
/// mount so the credentials never appear in config or container metadata.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_FILE")]
|
||||
sandbox_proxy_auth_file: Option<String>,
|
||||
|
||||
/// Explicit acknowledgement (`true`) that the proxy credential is sent
|
||||
/// as cleartext Basic auth over the plain-TCP connection to the http://
|
||||
/// proxy. Required when `--sandbox-proxy-auth-file` is set.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE")]
|
||||
sandbox_proxy_auth_allow_insecure: Option<bool>,
|
||||
|
||||
/// Send the destination hostname in CONNECT requests to the corporate
|
||||
/// proxy instead of a validated IP. Only for proxies whose ACLs filter
|
||||
/// on hostnames: the proxy then resolves the name itself, so sandbox
|
||||
/// SSRF/`allowed_ips` validation no longer binds the connection.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")]
|
||||
sandbox_proxy_connect_by_hostname: Option<bool>,
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("main_unix.rs");
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)),
|
||||
)
|
||||
.init();
|
||||
|
||||
let driver = PodmanComputeDriver::new(PodmanComputeConfig {
|
||||
socket_path: args.podman_socket,
|
||||
default_image: args.sandbox_image.unwrap_or_default(),
|
||||
image_pull_policy: args.sandbox_image_pull_policy,
|
||||
grpc_endpoint: args.grpc_endpoint.unwrap_or_default(),
|
||||
gateway_port: args.gateway_port,
|
||||
host_gateway_ip: args
|
||||
.host_gateway_ip
|
||||
.unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip),
|
||||
sandbox_ssh_socket_path: args.sandbox_ssh_socket_path,
|
||||
network_name: args.network_name,
|
||||
stop_timeout_secs: args.stop_timeout,
|
||||
supervisor_image: args
|
||||
.supervisor_image
|
||||
.unwrap_or_else(openshell_core::config::default_supervisor_image),
|
||||
guest_tls_ca: args.podman_tls_ca,
|
||||
guest_tls_cert: args.podman_tls_cert,
|
||||
guest_tls_key: args.podman_tls_key,
|
||||
sandbox_pids_limit: args.sandbox_pids_limit,
|
||||
https_proxy: args.sandbox_https_proxy,
|
||||
no_proxy: args.sandbox_no_proxy,
|
||||
proxy_auth_file: args.sandbox_proxy_auth_file,
|
||||
proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure,
|
||||
proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname,
|
||||
..PodmanComputeConfig::default()
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()?;
|
||||
|
||||
info!(address = %args.bind_address, "Starting Podman compute driver");
|
||||
tonic::transport::Server::builder()
|
||||
.add_service(ComputeDriverServer::new(ComputeDriverService::new(driver)))
|
||||
.serve_with_shutdown(args.bind_address, async {
|
||||
tokio::signal::ctrl_c().await.ok();
|
||||
info!("Received shutdown signal, draining in-flight requests");
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("main.rs");
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
include!("main_win.rs");
|
||||
@@ -1,185 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use clap::Parser;
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
use openshell_core::VERSION;
|
||||
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
|
||||
use openshell_driver_podman::config::{
|
||||
DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT,
|
||||
ImagePullPolicy,
|
||||
};
|
||||
use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "openshell-driver-podman")]
|
||||
#[command(version = VERSION)]
|
||||
struct Args {
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_COMPUTE_DRIVER_BIND",
|
||||
default_value = "127.0.0.1:50061"
|
||||
)]
|
||||
bind_address: SocketAddr,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")]
|
||||
log_level: String,
|
||||
|
||||
/// Path to the Podman API Unix socket.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_SOCKET")]
|
||||
podman_socket: Option<PathBuf>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")]
|
||||
sandbox_image: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY",
|
||||
default_value_t = ImagePullPolicy::Missing
|
||||
)]
|
||||
sandbox_image_pull_policy: ImagePullPolicy,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
|
||||
grpc_endpoint: Option<String>,
|
||||
|
||||
/// Port the gateway server is listening on.
|
||||
///
|
||||
/// Used when `--grpc-endpoint` is not set to auto-detect the endpoint
|
||||
/// that sandbox containers dial back to.
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_GATEWAY_PORT",
|
||||
default_value_t = openshell_core::config::DEFAULT_SERVER_PORT
|
||||
)]
|
||||
gateway_port: u16,
|
||||
|
||||
/// Host gateway IP used for sandbox host aliases.
|
||||
///
|
||||
/// Empty uses Podman's `host-gateway` resolver.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_HOST_GATEWAY_IP")]
|
||||
host_gateway_ip: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH",
|
||||
default_value = openshell_core::container_paths::SSH_SOCKET_PATH
|
||||
)]
|
||||
sandbox_ssh_socket_path: String,
|
||||
|
||||
/// Podman bridge network name.
|
||||
#[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)]
|
||||
network_name: String,
|
||||
|
||||
/// Container stop timeout in seconds (SIGTERM → SIGKILL).
|
||||
#[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)]
|
||||
stop_timeout: u32,
|
||||
|
||||
/// Container cgroup PID limit for sandbox containers. Set 0 to inherit
|
||||
/// Podman's runtime/default PID limit.
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SANDBOX_PIDS_LIMIT",
|
||||
default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT
|
||||
)]
|
||||
sandbox_pids_limit: i64,
|
||||
|
||||
/// OCI image containing the openshell-sandbox supervisor binary.
|
||||
#[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")]
|
||||
supervisor_image: Option<String>,
|
||||
|
||||
/// Host path to the CA certificate for sandbox mTLS.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_TLS_CA")]
|
||||
podman_tls_ca: Option<PathBuf>,
|
||||
|
||||
/// Host path to the client certificate for sandbox mTLS.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_TLS_CERT")]
|
||||
podman_tls_cert: Option<PathBuf>,
|
||||
|
||||
/// Host path to the client private key for sandbox mTLS.
|
||||
#[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")]
|
||||
podman_tls_key: Option<PathBuf>,
|
||||
|
||||
/// Corporate forward proxy URL for the supervisor's upstream TLS dials,
|
||||
/// in explicit `http://host:port` form (scheme and port required).
|
||||
/// Credentials must not be embedded in the URL; use
|
||||
/// `--sandbox-proxy-auth-file` instead.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_HTTPS_PROXY")]
|
||||
sandbox_https_proxy: Option<String>,
|
||||
|
||||
/// Comma-separated `NO_PROXY` list injected alongside the proxy URL.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_NO_PROXY")]
|
||||
sandbox_no_proxy: Option<String>,
|
||||
|
||||
/// Path to a file containing the corporate proxy credentials as
|
||||
/// `user:pass`. Delivered to the supervisor through a root-only secret
|
||||
/// mount so the credentials never appear in config or container metadata.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_FILE")]
|
||||
sandbox_proxy_auth_file: Option<String>,
|
||||
|
||||
/// Explicit acknowledgement (`true`) that the proxy credential is sent
|
||||
/// as cleartext Basic auth over the plain-TCP connection to the http://
|
||||
/// proxy. Required when `--sandbox-proxy-auth-file` is set.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE")]
|
||||
sandbox_proxy_auth_allow_insecure: Option<bool>,
|
||||
|
||||
/// Send the destination hostname in CONNECT requests to the corporate
|
||||
/// proxy instead of a validated IP. Only for proxies whose ACLs filter
|
||||
/// on hostnames: the proxy then resolves the name itself, so sandbox
|
||||
/// SSRF/`allowed_ips` validation no longer binds the connection.
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")]
|
||||
sandbox_proxy_connect_by_hostname: Option<bool>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)),
|
||||
)
|
||||
.init();
|
||||
|
||||
let driver = PodmanComputeDriver::new(PodmanComputeConfig {
|
||||
socket_path: args.podman_socket,
|
||||
default_image: args.sandbox_image.unwrap_or_default(),
|
||||
image_pull_policy: args.sandbox_image_pull_policy,
|
||||
grpc_endpoint: args.grpc_endpoint.unwrap_or_default(),
|
||||
gateway_port: args.gateway_port,
|
||||
host_gateway_ip: args
|
||||
.host_gateway_ip
|
||||
.unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip),
|
||||
sandbox_ssh_socket_path: args.sandbox_ssh_socket_path,
|
||||
network_name: args.network_name,
|
||||
stop_timeout_secs: args.stop_timeout,
|
||||
supervisor_image: args
|
||||
.supervisor_image
|
||||
.unwrap_or_else(openshell_core::config::default_supervisor_image),
|
||||
guest_tls_ca: args.podman_tls_ca,
|
||||
guest_tls_cert: args.podman_tls_cert,
|
||||
guest_tls_key: args.podman_tls_key,
|
||||
sandbox_pids_limit: args.sandbox_pids_limit,
|
||||
https_proxy: args.sandbox_https_proxy,
|
||||
no_proxy: args.sandbox_no_proxy,
|
||||
proxy_auth_file: args.sandbox_proxy_auth_file,
|
||||
proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure,
|
||||
proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname,
|
||||
..PodmanComputeConfig::default()
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()?;
|
||||
|
||||
info!(address = %args.bind_address, "Starting Podman compute driver");
|
||||
tonic::transport::Server::builder()
|
||||
.add_service(ComputeDriverServer::new(ComputeDriverService::new(driver)))
|
||||
.serve_with_shutdown(args.bind_address, async {
|
||||
tokio::signal::ctrl_c().await.ok();
|
||||
info!("Received shutdown signal, draining in-flight requests");
|
||||
})
|
||||
.await
|
||||
.into_diagnostic()
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
fn main() {
|
||||
eprintln!("openshell-driver-podman is unsupported on Windows");
|
||||
std::process::exit(1);
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
[package]
|
||||
name = "openshell-driver-vm"
|
||||
autobins = false
|
||||
description = "MicroVM compute driver for OpenShell"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
@@ -12,11 +13,11 @@ repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_driver_vm"
|
||||
path = "src/lib.rs"
|
||||
path = "src/lib_entry.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "openshell-driver-vm"
|
||||
path = "src/main.rs"
|
||||
path = "src/main_entry.rs"
|
||||
|
||||
[target.'cfg(not(target_os = "windows"))'.dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
|
||||
@@ -1,10 +1,24 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn windows_unsupported() -> &'static str {
|
||||
"openshell-driver-vm is unsupported on Windows"
|
||||
}
|
||||
pub mod driver;
|
||||
mod embedded_runtime;
|
||||
mod ffi;
|
||||
pub mod gpu;
|
||||
pub mod lifecycle;
|
||||
mod nft_ruleset;
|
||||
pub mod otel_tracing;
|
||||
pub mod procguard;
|
||||
mod rootfs;
|
||||
mod runtime;
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("lib_unix.rs");
|
||||
pub use driver::{VmDriver, VmDriverConfig};
|
||||
pub use lifecycle::{
|
||||
BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason,
|
||||
LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult,
|
||||
RestoreContext,
|
||||
};
|
||||
pub use runtime::{
|
||||
VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces,
|
||||
configured_runtime_dir, run_vm,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("lib.rs");
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
include!("lib_win.rs");
|
||||
@@ -1,24 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
pub mod driver;
|
||||
mod embedded_runtime;
|
||||
mod ffi;
|
||||
pub mod gpu;
|
||||
pub mod lifecycle;
|
||||
mod nft_ruleset;
|
||||
pub mod otel_tracing;
|
||||
pub mod procguard;
|
||||
mod rootfs;
|
||||
mod runtime;
|
||||
|
||||
pub use driver::{VmDriver, VmDriverConfig};
|
||||
pub use lifecycle::{
|
||||
BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason,
|
||||
LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult,
|
||||
RestoreContext,
|
||||
};
|
||||
pub use runtime::{
|
||||
VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces,
|
||||
configured_runtime_dir, run_vm,
|
||||
};
|
||||
@@ -0,0 +1,6 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
pub fn windows_unsupported() -> &'static str {
|
||||
"openshell-driver-vm is unsupported on Windows"
|
||||
}
|
||||
@@ -1,11 +1,782 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn main() {
|
||||
eprintln!("openshell-driver-vm is unsupported on Windows");
|
||||
std::process::exit(1);
|
||||
use clap::Parser;
|
||||
use futures::Stream;
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use openshell_core::VERSION;
|
||||
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
|
||||
use openshell_driver_vm::otel_tracing::compute_driver_rpc_layer;
|
||||
#[cfg(target_os = "macos")]
|
||||
use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir};
|
||||
use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm};
|
||||
use std::io;
|
||||
use std::net::SocketAddr;
|
||||
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::net::{UnixListener, UnixStream};
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::prelude::*;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "openshell-driver-vm")]
|
||||
#[command(version = VERSION)]
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
struct Args {
|
||||
#[arg(long, hide = true, default_value_t = false)]
|
||||
internal_run_vm: bool,
|
||||
|
||||
#[arg(long = "vm-root-disk", hide = true, alias = "vm-rootfs")]
|
||||
vm_root_disk: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "vm-overlay-disk", hide = true)]
|
||||
vm_overlay_disk: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "vm-image-disk", hide = true)]
|
||||
vm_image_disk: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "vm-kernel-image", hide = true)]
|
||||
vm_kernel_image: Option<PathBuf>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_exec: Option<String>,
|
||||
|
||||
#[arg(long, hide = true, default_value = "/")]
|
||||
vm_workdir: String,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_env: Vec<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_console_output: Option<PathBuf>,
|
||||
|
||||
#[arg(long, hide = true, default_value_t = 2)]
|
||||
vm_vcpus: u8,
|
||||
|
||||
#[arg(long, hide = true, default_value_t = 2048)]
|
||||
vm_mem_mib: u32,
|
||||
|
||||
#[arg(long, hide = true, default_value_t = 1)]
|
||||
vm_krun_log_level: u32,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_BIND")]
|
||||
bind_address: Option<SocketAddr>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")]
|
||||
bind_socket: Option<PathBuf>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
expected_peer_pid: Option<u32>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_UNAUTHENTICATED_TCP",
|
||||
default_value_t = false
|
||||
)]
|
||||
allow_unauthenticated_tcp: bool,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_SAME_UID_PEER",
|
||||
default_value_t = false
|
||||
)]
|
||||
allow_same_uid_peer: bool,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")]
|
||||
log_level: String,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
|
||||
otlp_endpoint: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
|
||||
openshell_endpoint: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")]
|
||||
default_image: String,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_BOOTSTRAP_IMAGE", default_value = "")]
|
||||
bootstrap_image: String,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_VM_DRIVER_STATE_DIR",
|
||||
default_value = "target/openshell-vm-driver"
|
||||
)]
|
||||
state_dir: PathBuf,
|
||||
|
||||
#[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")]
|
||||
guest_tls_ca: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "guest-tls-cert", env = "OPENSHELL_VM_TLS_CERT")]
|
||||
guest_tls_cert: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")]
|
||||
guest_tls_key: Option<PathBuf>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)]
|
||||
krun_log_level: u32,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_DRIVER_VCPUS", default_value_t = 2)]
|
||||
vcpus: u8,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_DRIVER_MEM_MIB", default_value_t = 2048)]
|
||||
mem_mib: u32,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_OVERLAY_DISK_MIB", default_value_t = 4096)]
|
||||
overlay_disk_mib: u64,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_GPU")]
|
||||
gpu: bool,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_GPU_MEM_MIB", default_value_t = 8192)]
|
||||
gpu_mem_mib: u32,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)]
|
||||
gpu_vcpus: u8,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_SANDBOX_UID")]
|
||||
sandbox_uid: Option<u32>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")]
|
||||
sandbox_gid: Option<u32>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_backend: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_gpu_bdf: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_tap_device: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_guest_ip: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_host_ip: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_vsock_cid: Option<u32>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_guest_mac: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_gateway_port: Option<u16>,
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("main_unix.rs");
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
if args.internal_run_vm {
|
||||
// We intentionally defer procguard arming until `run_vm()` so
|
||||
// that the only arm is the one that knows how to clean up
|
||||
// gvproxy. Racing two watchers against the same parent-death
|
||||
// event causes the bare arm's `exit(1)` to win, skipping the
|
||||
// gvproxy cleanup and leaking the helper. The risk window
|
||||
// before `run_vm` arms procguard is ~a few syscalls long
|
||||
// (`build_vm_launch_config`, `configured_runtime_dir`), which
|
||||
// is negligible next to the parent gRPC server's uptime.
|
||||
maybe_reexec_internal_vm_with_runtime_env()?;
|
||||
let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?;
|
||||
run_vm(&config).map_err(|err| miette::miette!("{err}"))?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (tracer_provider, setup_error) =
|
||||
openshell_driver_vm::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
|
||||
tracing_subscriber::registry()
|
||||
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
|
||||
.with(tracing_subscriber::fmt::layer())
|
||||
.with(
|
||||
tracer_provider
|
||||
.as_ref()
|
||||
.map(openshell_driver_vm::otel_tracing::layer),
|
||||
)
|
||||
.init();
|
||||
if let Some(error) = setup_error {
|
||||
tracing::error!(%error, "OTLP exporting could not be started");
|
||||
} else if let Some(endpoint) = &args.otlp_endpoint {
|
||||
info!(endpoint, "OTLP exporting enabled");
|
||||
}
|
||||
|
||||
let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?;
|
||||
|
||||
// Arm procguard so that if the gateway is killed (SIGKILL or crash)
|
||||
// we also die. Without this the driver is reparented to init and
|
||||
// keeps its per-sandbox VM launchers alive forever. Launchers have
|
||||
// their own procguards (armed in `run_vm`) which cascade cleanup of
|
||||
// gvproxy and the libkrun worker the moment this driver exits.
|
||||
if let Err(err) = procguard::die_with_parent() {
|
||||
tracing::warn!(
|
||||
error = %err,
|
||||
"procguard arm failed; gateway crashes may orphan this driver"
|
||||
);
|
||||
}
|
||||
|
||||
let driver = VmDriver::new(VmDriverConfig {
|
||||
openshell_endpoint: args
|
||||
.openshell_endpoint
|
||||
.ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?,
|
||||
state_dir: args.state_dir.clone(),
|
||||
launcher_bin: None,
|
||||
default_image: args.default_image.clone(),
|
||||
bootstrap_image: args.bootstrap_image.clone(),
|
||||
log_level: args.log_level.clone(),
|
||||
krun_log_level: args.krun_log_level,
|
||||
vcpus: args.vcpus,
|
||||
mem_mib: args.mem_mib,
|
||||
overlay_disk_mib: args.overlay_disk_mib,
|
||||
guest_tls_ca: args.guest_tls_ca.clone(),
|
||||
guest_tls_cert: args.guest_tls_cert.clone(),
|
||||
guest_tls_key: args.guest_tls_key.clone(),
|
||||
gpu_enabled: args.gpu,
|
||||
gpu_mem_mib: args.gpu_mem_mib,
|
||||
gpu_vcpus: args.gpu_vcpus,
|
||||
sandbox_uid: args.sandbox_uid,
|
||||
sandbox_gid: args.sandbox_gid,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| miette::miette!("{err}"))?;
|
||||
|
||||
let result = match listen_mode {
|
||||
ComputeDriverListenMode::Unix {
|
||||
socket_path,
|
||||
expected_peer_pid,
|
||||
} => {
|
||||
prepare_compute_driver_socket(&socket_path).map_err(|err| miette::miette!("{err}"))?;
|
||||
|
||||
info!(socket = %socket_path.display(), "Starting vm compute driver");
|
||||
let listener = UnixListener::bind(&socket_path).into_diagnostic()?;
|
||||
restrict_socket_permissions(&socket_path).map_err(|err| miette::miette!("{err}"))?;
|
||||
let result = tonic::transport::Server::builder()
|
||||
.layer(compute_driver_rpc_layer())
|
||||
.add_service(ComputeDriverServer::new(driver))
|
||||
.serve_with_incoming_shutdown(
|
||||
AuthenticatedUnixIncoming::new(listener, expected_peer_pid),
|
||||
shutdown_signal(),
|
||||
)
|
||||
.await
|
||||
.into_diagnostic();
|
||||
let _ = std::fs::remove_file(&socket_path);
|
||||
result
|
||||
}
|
||||
ComputeDriverListenMode::Tcp(bind_address) => {
|
||||
info!(address = %bind_address, "Starting unauthenticated dev vm compute driver");
|
||||
tonic::transport::Server::builder()
|
||||
.layer(compute_driver_rpc_layer())
|
||||
.add_service(ComputeDriverServer::new(driver))
|
||||
.serve_with_shutdown(bind_address, shutdown_signal())
|
||||
.await
|
||||
.into_diagnostic()
|
||||
}
|
||||
};
|
||||
if let Some(provider) = &tracer_provider
|
||||
&& let Err(error) = provider.shutdown()
|
||||
{
|
||||
tracing::warn!(%error, "OTLP tracer provider shutdown failed");
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn shutdown_signal() {
|
||||
let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
.expect("install SIGTERM handler");
|
||||
tokio::select! {
|
||||
result = tokio::signal::ctrl_c() => {
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(%error, "failed to listen for Ctrl-C");
|
||||
}
|
||||
}
|
||||
_ = terminate.recv() => {}
|
||||
}
|
||||
info!("Shutdown signal received; stopping vm compute driver");
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum ComputeDriverListenMode {
|
||||
Unix {
|
||||
socket_path: PathBuf,
|
||||
expected_peer_pid: Option<u32>,
|
||||
},
|
||||
Tcp(SocketAddr),
|
||||
}
|
||||
|
||||
fn compute_driver_listen_mode(args: &Args) -> std::result::Result<ComputeDriverListenMode, String> {
|
||||
if let Some(socket_path) = args.bind_socket.clone() {
|
||||
if args.expected_peer_pid.is_none() && !args.allow_same_uid_peer {
|
||||
return Err(
|
||||
"--expected-peer-pid is required with --bind-socket; use --allow-same-uid-peer only for local development"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
return Ok(ComputeDriverListenMode::Unix {
|
||||
socket_path,
|
||||
expected_peer_pid: args.expected_peer_pid,
|
||||
});
|
||||
}
|
||||
|
||||
if !args.allow_unauthenticated_tcp {
|
||||
return Err(
|
||||
"--bind-socket is required; unauthenticated TCP mode is disabled unless --allow-unauthenticated-tcp is set for local development"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let Some(bind_address) = args.bind_address else {
|
||||
return Err("--bind-address is required with --allow-unauthenticated-tcp".to_string());
|
||||
};
|
||||
|
||||
Ok(ComputeDriverListenMode::Tcp(bind_address))
|
||||
}
|
||||
|
||||
fn prepare_compute_driver_socket(socket_path: &Path) -> std::result::Result<(), String> {
|
||||
let Some(parent) = socket_path.parent() else {
|
||||
return Err(format!(
|
||||
"vm compute driver socket path '{}' has no parent directory",
|
||||
socket_path.display()
|
||||
));
|
||||
};
|
||||
let expected_uid = current_euid();
|
||||
prepare_private_socket_dir(parent, expected_uid)?;
|
||||
remove_stale_socket(socket_path, expected_uid)
|
||||
}
|
||||
|
||||
fn current_euid() -> u32 {
|
||||
rustix::process::geteuid().as_raw()
|
||||
}
|
||||
|
||||
fn prepare_private_socket_dir(
|
||||
socket_dir: &Path,
|
||||
expected_uid: u32,
|
||||
) -> std::result::Result<(), String> {
|
||||
std::fs::create_dir_all(socket_dir)
|
||||
.map_err(|err| format!("create socket dir {}: {err}", socket_dir.display()))?;
|
||||
let metadata = std::fs::symlink_metadata(socket_dir)
|
||||
.map_err(|err| format!("stat socket dir {}: {err}", socket_dir.display()))?;
|
||||
let file_type = metadata.file_type();
|
||||
if file_type.is_symlink() {
|
||||
return Err(format!(
|
||||
"socket dir {} is a symlink; refusing to use it",
|
||||
socket_dir.display()
|
||||
));
|
||||
}
|
||||
if !file_type.is_dir() {
|
||||
return Err(format!(
|
||||
"socket dir {} is not a directory",
|
||||
socket_dir.display()
|
||||
));
|
||||
}
|
||||
if metadata.uid() != expected_uid {
|
||||
return Err(format!(
|
||||
"socket dir {} is owned by uid {} but current euid is {}",
|
||||
socket_dir.display(),
|
||||
metadata.uid(),
|
||||
expected_uid
|
||||
));
|
||||
}
|
||||
std::fs::set_permissions(socket_dir, std::fs::Permissions::from_mode(0o700))
|
||||
.map_err(|err| format!("chmod socket dir {}: {err}", socket_dir.display()))
|
||||
}
|
||||
|
||||
fn remove_stale_socket(socket_path: &Path, expected_uid: u32) -> std::result::Result<(), String> {
|
||||
let metadata = match std::fs::symlink_metadata(socket_path) {
|
||||
Ok(metadata) => metadata,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => return Err(format!("stat socket {}: {err}", socket_path.display())),
|
||||
};
|
||||
let file_type = metadata.file_type();
|
||||
if file_type.is_symlink() {
|
||||
return Err(format!(
|
||||
"socket {} is a symlink; refusing to remove it",
|
||||
socket_path.display()
|
||||
));
|
||||
}
|
||||
if metadata.uid() != expected_uid {
|
||||
return Err(format!(
|
||||
"socket {} is owned by uid {} but current euid is {}",
|
||||
socket_path.display(),
|
||||
metadata.uid(),
|
||||
expected_uid
|
||||
));
|
||||
}
|
||||
if !file_type.is_socket() {
|
||||
return Err(format!(
|
||||
"socket path {} exists but is not a Unix socket",
|
||||
socket_path.display()
|
||||
));
|
||||
}
|
||||
std::fs::remove_file(socket_path)
|
||||
.map_err(|err| format!("remove stale socket {}: {err}", socket_path.display()))
|
||||
}
|
||||
|
||||
fn restrict_socket_permissions(socket_path: &Path) -> std::result::Result<(), String> {
|
||||
std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))
|
||||
.map_err(|err| format!("chmod socket {}: {err}", socket_path.display()))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct PeerCredentials {
|
||||
uid: u32,
|
||||
pid: Option<i32>,
|
||||
}
|
||||
|
||||
fn peer_credentials(stream: &UnixStream) -> std::result::Result<PeerCredentials, String> {
|
||||
let credentials = stream
|
||||
.peer_cred()
|
||||
.map_err(|err| format!("read peer credentials: {err}"))?;
|
||||
Ok(PeerCredentials {
|
||||
uid: credentials.uid(),
|
||||
pid: credentials.pid(),
|
||||
})
|
||||
}
|
||||
|
||||
fn authorize_peer_credentials(
|
||||
peer: PeerCredentials,
|
||||
driver_uid: u32,
|
||||
gateway_pid: Option<u32>,
|
||||
) -> std::result::Result<(), String> {
|
||||
if peer.uid != driver_uid {
|
||||
return Err(format!(
|
||||
"peer uid {} does not match current euid {}",
|
||||
peer.uid, driver_uid
|
||||
));
|
||||
}
|
||||
let Some(gateway_pid) = gateway_pid else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(peer_process_id) = peer.pid.and_then(|pid| u32::try_from(pid).ok()) else {
|
||||
return Err(format!(
|
||||
"peer pid is unavailable; expected gateway pid {gateway_pid}"
|
||||
));
|
||||
};
|
||||
if peer_process_id != gateway_pid {
|
||||
return Err(format!(
|
||||
"peer pid {peer_process_id} does not match expected gateway pid {gateway_pid}"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct AuthenticatedUnixIncoming {
|
||||
listener: UnixListener,
|
||||
expected_uid: u32,
|
||||
expected_peer_pid: Option<u32>,
|
||||
}
|
||||
|
||||
impl AuthenticatedUnixIncoming {
|
||||
fn new(listener: UnixListener, expected_peer_pid: Option<u32>) -> Self {
|
||||
Self {
|
||||
listener,
|
||||
expected_uid: current_euid(),
|
||||
expected_peer_pid,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for AuthenticatedUnixIncoming {
|
||||
type Item = io::Result<UnixStream>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.get_mut();
|
||||
loop {
|
||||
match this.listener.poll_accept(cx) {
|
||||
Poll::Ready(Ok((stream, _addr))) => {
|
||||
let authorized = peer_credentials(&stream).and_then(|peer| {
|
||||
authorize_peer_credentials(peer, this.expected_uid, this.expected_peer_pid)
|
||||
});
|
||||
match authorized {
|
||||
Ok(()) => return Poll::Ready(Some(Ok(stream))),
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
error = %err,
|
||||
"rejected vm compute driver UDS client"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_vm_launch_config(args: &Args) -> std::result::Result<VmLaunchConfig, String> {
|
||||
let root_disk = args
|
||||
.vm_root_disk
|
||||
.clone()
|
||||
.ok_or_else(|| "--vm-root-disk is required in internal VM mode".to_string())?;
|
||||
let overlay_disk = args
|
||||
.vm_overlay_disk
|
||||
.clone()
|
||||
.ok_or_else(|| "--vm-overlay-disk is required in internal VM mode".to_string())?;
|
||||
let image_disk = args.vm_image_disk.clone();
|
||||
let exec_path = args
|
||||
.vm_exec
|
||||
.clone()
|
||||
.ok_or_else(|| "--vm-exec is required in internal VM mode".to_string())?;
|
||||
let console_output = args
|
||||
.vm_console_output
|
||||
.clone()
|
||||
.ok_or_else(|| "--vm-console-output is required in internal VM mode".to_string())?;
|
||||
|
||||
let backend = match args.vm_backend.as_deref() {
|
||||
Some("qemu") => VmBackend::Qemu,
|
||||
Some("libkrun") | None => VmBackend::Libkrun,
|
||||
Some(other) => return Err(format!("unknown VM backend: {other}")),
|
||||
};
|
||||
|
||||
Ok(VmLaunchConfig {
|
||||
root_disk,
|
||||
overlay_disk,
|
||||
image_disk,
|
||||
kernel_image: args.vm_kernel_image.clone(),
|
||||
vcpus: args.vm_vcpus,
|
||||
mem_mib: args.vm_mem_mib,
|
||||
exec_path,
|
||||
args: Vec::new(),
|
||||
env: args.vm_env.clone(),
|
||||
workdir: args.vm_workdir.clone(),
|
||||
log_level: args.vm_krun_log_level,
|
||||
console_output,
|
||||
backend,
|
||||
gpu_bdf: args.vm_gpu_bdf.clone(),
|
||||
tap_device: args.vm_tap_device.clone(),
|
||||
guest_ip: args.vm_guest_ip.clone(),
|
||||
host_ip: args.vm_host_ip.clone(),
|
||||
vsock_cid: args.vm_vsock_cid,
|
||||
guest_mac: args.vm_guest_mac.clone(),
|
||||
gateway_port: args.vm_gateway_port,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> {
|
||||
use std::os::unix::process::CommandExt as _;
|
||||
|
||||
const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC";
|
||||
|
||||
if std::env::var_os(REEXEC_ENV).is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let runtime_dir = configured_runtime_dir().map_err(|err| miette::miette!("{err}"))?;
|
||||
let runtime_str = runtime_dir.to_string_lossy();
|
||||
let needs_reexec = std::env::var_os("DYLD_LIBRARY_PATH")
|
||||
.is_none_or(|value| !value.to_string_lossy().contains(runtime_str.as_ref()));
|
||||
if !needs_reexec {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut dyld_paths = vec![runtime_dir.clone()];
|
||||
if let Some(existing) = std::env::var_os("DYLD_LIBRARY_PATH") {
|
||||
dyld_paths.extend(std::env::split_paths(&existing));
|
||||
}
|
||||
let joined = std::env::join_paths(&dyld_paths)
|
||||
.map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?;
|
||||
let exe = std::env::current_exe().into_diagnostic()?;
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
|
||||
// Use execvp() so the current process is *replaced* by the re-exec'd
|
||||
// binary — no wrapper process sits between the compute driver and
|
||||
// the actually-running VM launcher. That avoids two problems:
|
||||
// 1. An extra process level that survives SIGKILL of the driver
|
||||
// (the wrapper was reparenting the re-exec'd child to init).
|
||||
// 2. Signal forwarding: with a wrapper, a SIGTERM to the wrapper
|
||||
// doesn't reach the child unless we hand-roll forwarding.
|
||||
// After exec, the child inherits our PID and our procguard arming.
|
||||
let err = std::process::Command::new(exe)
|
||||
.args(&args)
|
||||
.env("DYLD_LIBRARY_PATH", &joined)
|
||||
.env(VM_RUNTIME_DIR_ENV, runtime_dir)
|
||||
.env(REEXEC_ENV, "1")
|
||||
.exec();
|
||||
// `exec()` only returns on failure.
|
||||
Err(miette::miette!("failed to re-exec with runtime env: {err}"))
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
// Signature must match the macOS variant which can fail.
|
||||
#[allow(clippy::unnecessary_wraps)]
|
||||
fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
Args, ComputeDriverListenMode, PeerCredentials, authorize_peer_credentials,
|
||||
compute_driver_listen_mode,
|
||||
};
|
||||
use clap::Parser;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_accepts_matching_uid_and_pid() {
|
||||
authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1000,
|
||||
pid: Some(42),
|
||||
},
|
||||
1000,
|
||||
Some(42),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_rejects_wrong_pid() {
|
||||
let err = authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1000,
|
||||
pid: Some(7),
|
||||
},
|
||||
1000,
|
||||
Some(42),
|
||||
)
|
||||
.expect_err("wrong pid should be rejected");
|
||||
assert!(err.contains("does not match expected gateway pid"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_rejects_wrong_uid() {
|
||||
let err = authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1001,
|
||||
pid: Some(42),
|
||||
},
|
||||
1000,
|
||||
Some(42),
|
||||
)
|
||||
.expect_err("wrong uid should be rejected");
|
||||
assert!(err.contains("does not match current euid"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_rejects_missing_pid_when_expected() {
|
||||
let err = authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1000,
|
||||
pid: None,
|
||||
},
|
||||
1000,
|
||||
Some(42),
|
||||
)
|
||||
.expect_err("missing pid should be rejected");
|
||||
assert!(err.contains("peer pid is unavailable"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_accepts_matching_uid_without_expected_pid() {
|
||||
authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1000,
|
||||
pid: None,
|
||||
},
|
||||
1000,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_rejects_default_tcp() {
|
||||
let args = Args::parse_from(["openshell-driver-vm"]);
|
||||
let err = compute_driver_listen_mode(&args).expect_err("default TCP should be disabled");
|
||||
assert!(err.contains("--bind-socket is required"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_gateway_otlp_endpoint() {
|
||||
let args = Args::try_parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--otlp-endpoint",
|
||||
"http://127.0.0.1:4317",
|
||||
]);
|
||||
assert!(
|
||||
args.is_ok(),
|
||||
"VM driver should accept the gateway OTLP endpoint"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_rejects_bind_address_without_tcp_opt_in() {
|
||||
let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]);
|
||||
let err =
|
||||
compute_driver_listen_mode(&args).expect_err("TCP bind should require explicit opt-in");
|
||||
assert!(err.contains("--allow-unauthenticated-tcp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_requires_bind_address_with_tcp_opt_in() {
|
||||
let args = Args::parse_from(["openshell-driver-vm", "--allow-unauthenticated-tcp"]);
|
||||
let err =
|
||||
compute_driver_listen_mode(&args).expect_err("TCP opt-in should require an address");
|
||||
assert!(err.contains("--bind-address is required"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_accepts_explicit_unauthenticated_tcp() {
|
||||
let args = Args::parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--allow-unauthenticated-tcp",
|
||||
"--bind-address",
|
||||
"127.0.0.1:50061",
|
||||
]);
|
||||
assert_eq!(
|
||||
compute_driver_listen_mode(&args).unwrap(),
|
||||
ComputeDriverListenMode::Tcp("127.0.0.1:50061".parse().unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_requires_expected_peer_pid_for_uds() {
|
||||
let args = Args::parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--bind-socket",
|
||||
"/tmp/compute-driver.sock",
|
||||
]);
|
||||
let err = compute_driver_listen_mode(&args)
|
||||
.expect_err("UDS should require gateway peer pid by default");
|
||||
assert!(err.contains("--expected-peer-pid is required"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_accepts_uds_with_expected_peer_pid() {
|
||||
let args = Args::parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--bind-socket",
|
||||
"/tmp/compute-driver.sock",
|
||||
"--expected-peer-pid",
|
||||
"42",
|
||||
]);
|
||||
assert_eq!(
|
||||
compute_driver_listen_mode(&args).unwrap(),
|
||||
ComputeDriverListenMode::Unix {
|
||||
socket_path: PathBuf::from("/tmp/compute-driver.sock"),
|
||||
expected_peer_pid: Some(42),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_accepts_explicit_same_uid_uds_dev_mode() {
|
||||
let args = Args::parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--bind-socket",
|
||||
"/tmp/compute-driver.sock",
|
||||
"--allow-same-uid-peer",
|
||||
]);
|
||||
assert_eq!(
|
||||
compute_driver_listen_mode(&args).unwrap(),
|
||||
ComputeDriverListenMode::Unix {
|
||||
socket_path: PathBuf::from("/tmp/compute-driver.sock"),
|
||||
expected_peer_pid: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("main.rs");
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
include!("main_win.rs");
|
||||
@@ -1,782 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use clap::Parser;
|
||||
use futures::Stream;
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use openshell_core::VERSION;
|
||||
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
|
||||
use openshell_driver_vm::otel_tracing::compute_driver_rpc_layer;
|
||||
#[cfg(target_os = "macos")]
|
||||
use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir};
|
||||
use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm};
|
||||
use std::io;
|
||||
use std::net::SocketAddr;
|
||||
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::net::{UnixListener, UnixStream};
|
||||
use tracing::info;
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::prelude::*;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "openshell-driver-vm")]
|
||||
#[command(version = VERSION)]
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
struct Args {
|
||||
#[arg(long, hide = true, default_value_t = false)]
|
||||
internal_run_vm: bool,
|
||||
|
||||
#[arg(long = "vm-root-disk", hide = true, alias = "vm-rootfs")]
|
||||
vm_root_disk: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "vm-overlay-disk", hide = true)]
|
||||
vm_overlay_disk: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "vm-image-disk", hide = true)]
|
||||
vm_image_disk: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "vm-kernel-image", hide = true)]
|
||||
vm_kernel_image: Option<PathBuf>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_exec: Option<String>,
|
||||
|
||||
#[arg(long, hide = true, default_value = "/")]
|
||||
vm_workdir: String,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_env: Vec<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_console_output: Option<PathBuf>,
|
||||
|
||||
#[arg(long, hide = true, default_value_t = 2)]
|
||||
vm_vcpus: u8,
|
||||
|
||||
#[arg(long, hide = true, default_value_t = 2048)]
|
||||
vm_mem_mib: u32,
|
||||
|
||||
#[arg(long, hide = true, default_value_t = 1)]
|
||||
vm_krun_log_level: u32,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_BIND")]
|
||||
bind_address: Option<SocketAddr>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")]
|
||||
bind_socket: Option<PathBuf>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
expected_peer_pid: Option<u32>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_UNAUTHENTICATED_TCP",
|
||||
default_value_t = false
|
||||
)]
|
||||
allow_unauthenticated_tcp: bool,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_SAME_UID_PEER",
|
||||
default_value_t = false
|
||||
)]
|
||||
allow_same_uid_peer: bool,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")]
|
||||
log_level: String,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")]
|
||||
otlp_endpoint: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
|
||||
openshell_endpoint: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")]
|
||||
default_image: String,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_BOOTSTRAP_IMAGE", default_value = "")]
|
||||
bootstrap_image: String,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_VM_DRIVER_STATE_DIR",
|
||||
default_value = "target/openshell-vm-driver"
|
||||
)]
|
||||
state_dir: PathBuf,
|
||||
|
||||
#[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")]
|
||||
guest_tls_ca: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "guest-tls-cert", env = "OPENSHELL_VM_TLS_CERT")]
|
||||
guest_tls_cert: Option<PathBuf>,
|
||||
|
||||
#[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")]
|
||||
guest_tls_key: Option<PathBuf>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)]
|
||||
krun_log_level: u32,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_DRIVER_VCPUS", default_value_t = 2)]
|
||||
vcpus: u8,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_DRIVER_MEM_MIB", default_value_t = 2048)]
|
||||
mem_mib: u32,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_OVERLAY_DISK_MIB", default_value_t = 4096)]
|
||||
overlay_disk_mib: u64,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_GPU")]
|
||||
gpu: bool,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_GPU_MEM_MIB", default_value_t = 8192)]
|
||||
gpu_mem_mib: u32,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)]
|
||||
gpu_vcpus: u8,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_SANDBOX_UID")]
|
||||
sandbox_uid: Option<u32>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")]
|
||||
sandbox_gid: Option<u32>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_backend: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_gpu_bdf: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_tap_device: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_guest_ip: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_host_ip: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_vsock_cid: Option<u32>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_guest_mac: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_gateway_port: Option<u16>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> Result<()> {
|
||||
let args = Args::parse();
|
||||
if args.internal_run_vm {
|
||||
// We intentionally defer procguard arming until `run_vm()` so
|
||||
// that the only arm is the one that knows how to clean up
|
||||
// gvproxy. Racing two watchers against the same parent-death
|
||||
// event causes the bare arm's `exit(1)` to win, skipping the
|
||||
// gvproxy cleanup and leaking the helper. The risk window
|
||||
// before `run_vm` arms procguard is ~a few syscalls long
|
||||
// (`build_vm_launch_config`, `configured_runtime_dir`), which
|
||||
// is negligible next to the parent gRPC server's uptime.
|
||||
maybe_reexec_internal_vm_with_runtime_env()?;
|
||||
let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?;
|
||||
run_vm(&config).map_err(|err| miette::miette!("{err}"))?;
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let (tracer_provider, setup_error) =
|
||||
openshell_driver_vm::otel_tracing::provider_for(args.otlp_endpoint.as_deref());
|
||||
tracing_subscriber::registry()
|
||||
.with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)))
|
||||
.with(tracing_subscriber::fmt::layer())
|
||||
.with(
|
||||
tracer_provider
|
||||
.as_ref()
|
||||
.map(openshell_driver_vm::otel_tracing::layer),
|
||||
)
|
||||
.init();
|
||||
if let Some(error) = setup_error {
|
||||
tracing::error!(%error, "OTLP exporting could not be started");
|
||||
} else if let Some(endpoint) = &args.otlp_endpoint {
|
||||
info!(endpoint, "OTLP exporting enabled");
|
||||
}
|
||||
|
||||
let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?;
|
||||
|
||||
// Arm procguard so that if the gateway is killed (SIGKILL or crash)
|
||||
// we also die. Without this the driver is reparented to init and
|
||||
// keeps its per-sandbox VM launchers alive forever. Launchers have
|
||||
// their own procguards (armed in `run_vm`) which cascade cleanup of
|
||||
// gvproxy and the libkrun worker the moment this driver exits.
|
||||
if let Err(err) = procguard::die_with_parent() {
|
||||
tracing::warn!(
|
||||
error = %err,
|
||||
"procguard arm failed; gateway crashes may orphan this driver"
|
||||
);
|
||||
}
|
||||
|
||||
let driver = VmDriver::new(VmDriverConfig {
|
||||
openshell_endpoint: args
|
||||
.openshell_endpoint
|
||||
.ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?,
|
||||
state_dir: args.state_dir.clone(),
|
||||
launcher_bin: None,
|
||||
default_image: args.default_image.clone(),
|
||||
bootstrap_image: args.bootstrap_image.clone(),
|
||||
log_level: args.log_level.clone(),
|
||||
krun_log_level: args.krun_log_level,
|
||||
vcpus: args.vcpus,
|
||||
mem_mib: args.mem_mib,
|
||||
overlay_disk_mib: args.overlay_disk_mib,
|
||||
guest_tls_ca: args.guest_tls_ca.clone(),
|
||||
guest_tls_cert: args.guest_tls_cert.clone(),
|
||||
guest_tls_key: args.guest_tls_key.clone(),
|
||||
gpu_enabled: args.gpu,
|
||||
gpu_mem_mib: args.gpu_mem_mib,
|
||||
gpu_vcpus: args.gpu_vcpus,
|
||||
sandbox_uid: args.sandbox_uid,
|
||||
sandbox_gid: args.sandbox_gid,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| miette::miette!("{err}"))?;
|
||||
|
||||
let result = match listen_mode {
|
||||
ComputeDriverListenMode::Unix {
|
||||
socket_path,
|
||||
expected_peer_pid,
|
||||
} => {
|
||||
prepare_compute_driver_socket(&socket_path).map_err(|err| miette::miette!("{err}"))?;
|
||||
|
||||
info!(socket = %socket_path.display(), "Starting vm compute driver");
|
||||
let listener = UnixListener::bind(&socket_path).into_diagnostic()?;
|
||||
restrict_socket_permissions(&socket_path).map_err(|err| miette::miette!("{err}"))?;
|
||||
let result = tonic::transport::Server::builder()
|
||||
.layer(compute_driver_rpc_layer())
|
||||
.add_service(ComputeDriverServer::new(driver))
|
||||
.serve_with_incoming_shutdown(
|
||||
AuthenticatedUnixIncoming::new(listener, expected_peer_pid),
|
||||
shutdown_signal(),
|
||||
)
|
||||
.await
|
||||
.into_diagnostic();
|
||||
let _ = std::fs::remove_file(&socket_path);
|
||||
result
|
||||
}
|
||||
ComputeDriverListenMode::Tcp(bind_address) => {
|
||||
info!(address = %bind_address, "Starting unauthenticated dev vm compute driver");
|
||||
tonic::transport::Server::builder()
|
||||
.layer(compute_driver_rpc_layer())
|
||||
.add_service(ComputeDriverServer::new(driver))
|
||||
.serve_with_shutdown(bind_address, shutdown_signal())
|
||||
.await
|
||||
.into_diagnostic()
|
||||
}
|
||||
};
|
||||
if let Some(provider) = &tracer_provider
|
||||
&& let Err(error) = provider.shutdown()
|
||||
{
|
||||
tracing::warn!(%error, "OTLP tracer provider shutdown failed");
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
async fn shutdown_signal() {
|
||||
let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
.expect("install SIGTERM handler");
|
||||
tokio::select! {
|
||||
result = tokio::signal::ctrl_c() => {
|
||||
if let Err(error) = result {
|
||||
tracing::warn!(%error, "failed to listen for Ctrl-C");
|
||||
}
|
||||
}
|
||||
_ = terminate.recv() => {}
|
||||
}
|
||||
info!("Shutdown signal received; stopping vm compute driver");
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
enum ComputeDriverListenMode {
|
||||
Unix {
|
||||
socket_path: PathBuf,
|
||||
expected_peer_pid: Option<u32>,
|
||||
},
|
||||
Tcp(SocketAddr),
|
||||
}
|
||||
|
||||
fn compute_driver_listen_mode(args: &Args) -> std::result::Result<ComputeDriverListenMode, String> {
|
||||
if let Some(socket_path) = args.bind_socket.clone() {
|
||||
if args.expected_peer_pid.is_none() && !args.allow_same_uid_peer {
|
||||
return Err(
|
||||
"--expected-peer-pid is required with --bind-socket; use --allow-same-uid-peer only for local development"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
return Ok(ComputeDriverListenMode::Unix {
|
||||
socket_path,
|
||||
expected_peer_pid: args.expected_peer_pid,
|
||||
});
|
||||
}
|
||||
|
||||
if !args.allow_unauthenticated_tcp {
|
||||
return Err(
|
||||
"--bind-socket is required; unauthenticated TCP mode is disabled unless --allow-unauthenticated-tcp is set for local development"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let Some(bind_address) = args.bind_address else {
|
||||
return Err("--bind-address is required with --allow-unauthenticated-tcp".to_string());
|
||||
};
|
||||
|
||||
Ok(ComputeDriverListenMode::Tcp(bind_address))
|
||||
}
|
||||
|
||||
fn prepare_compute_driver_socket(socket_path: &Path) -> std::result::Result<(), String> {
|
||||
let Some(parent) = socket_path.parent() else {
|
||||
return Err(format!(
|
||||
"vm compute driver socket path '{}' has no parent directory",
|
||||
socket_path.display()
|
||||
));
|
||||
};
|
||||
let expected_uid = current_euid();
|
||||
prepare_private_socket_dir(parent, expected_uid)?;
|
||||
remove_stale_socket(socket_path, expected_uid)
|
||||
}
|
||||
|
||||
fn current_euid() -> u32 {
|
||||
rustix::process::geteuid().as_raw()
|
||||
}
|
||||
|
||||
fn prepare_private_socket_dir(
|
||||
socket_dir: &Path,
|
||||
expected_uid: u32,
|
||||
) -> std::result::Result<(), String> {
|
||||
std::fs::create_dir_all(socket_dir)
|
||||
.map_err(|err| format!("create socket dir {}: {err}", socket_dir.display()))?;
|
||||
let metadata = std::fs::symlink_metadata(socket_dir)
|
||||
.map_err(|err| format!("stat socket dir {}: {err}", socket_dir.display()))?;
|
||||
let file_type = metadata.file_type();
|
||||
if file_type.is_symlink() {
|
||||
return Err(format!(
|
||||
"socket dir {} is a symlink; refusing to use it",
|
||||
socket_dir.display()
|
||||
));
|
||||
}
|
||||
if !file_type.is_dir() {
|
||||
return Err(format!(
|
||||
"socket dir {} is not a directory",
|
||||
socket_dir.display()
|
||||
));
|
||||
}
|
||||
if metadata.uid() != expected_uid {
|
||||
return Err(format!(
|
||||
"socket dir {} is owned by uid {} but current euid is {}",
|
||||
socket_dir.display(),
|
||||
metadata.uid(),
|
||||
expected_uid
|
||||
));
|
||||
}
|
||||
std::fs::set_permissions(socket_dir, std::fs::Permissions::from_mode(0o700))
|
||||
.map_err(|err| format!("chmod socket dir {}: {err}", socket_dir.display()))
|
||||
}
|
||||
|
||||
fn remove_stale_socket(socket_path: &Path, expected_uid: u32) -> std::result::Result<(), String> {
|
||||
let metadata = match std::fs::symlink_metadata(socket_path) {
|
||||
Ok(metadata) => metadata,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => return Err(format!("stat socket {}: {err}", socket_path.display())),
|
||||
};
|
||||
let file_type = metadata.file_type();
|
||||
if file_type.is_symlink() {
|
||||
return Err(format!(
|
||||
"socket {} is a symlink; refusing to remove it",
|
||||
socket_path.display()
|
||||
));
|
||||
}
|
||||
if metadata.uid() != expected_uid {
|
||||
return Err(format!(
|
||||
"socket {} is owned by uid {} but current euid is {}",
|
||||
socket_path.display(),
|
||||
metadata.uid(),
|
||||
expected_uid
|
||||
));
|
||||
}
|
||||
if !file_type.is_socket() {
|
||||
return Err(format!(
|
||||
"socket path {} exists but is not a Unix socket",
|
||||
socket_path.display()
|
||||
));
|
||||
}
|
||||
std::fs::remove_file(socket_path)
|
||||
.map_err(|err| format!("remove stale socket {}: {err}", socket_path.display()))
|
||||
}
|
||||
|
||||
fn restrict_socket_permissions(socket_path: &Path) -> std::result::Result<(), String> {
|
||||
std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600))
|
||||
.map_err(|err| format!("chmod socket {}: {err}", socket_path.display()))
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct PeerCredentials {
|
||||
uid: u32,
|
||||
pid: Option<i32>,
|
||||
}
|
||||
|
||||
fn peer_credentials(stream: &UnixStream) -> std::result::Result<PeerCredentials, String> {
|
||||
let credentials = stream
|
||||
.peer_cred()
|
||||
.map_err(|err| format!("read peer credentials: {err}"))?;
|
||||
Ok(PeerCredentials {
|
||||
uid: credentials.uid(),
|
||||
pid: credentials.pid(),
|
||||
})
|
||||
}
|
||||
|
||||
fn authorize_peer_credentials(
|
||||
peer: PeerCredentials,
|
||||
driver_uid: u32,
|
||||
gateway_pid: Option<u32>,
|
||||
) -> std::result::Result<(), String> {
|
||||
if peer.uid != driver_uid {
|
||||
return Err(format!(
|
||||
"peer uid {} does not match current euid {}",
|
||||
peer.uid, driver_uid
|
||||
));
|
||||
}
|
||||
let Some(gateway_pid) = gateway_pid else {
|
||||
return Ok(());
|
||||
};
|
||||
let Some(peer_process_id) = peer.pid.and_then(|pid| u32::try_from(pid).ok()) else {
|
||||
return Err(format!(
|
||||
"peer pid is unavailable; expected gateway pid {gateway_pid}"
|
||||
));
|
||||
};
|
||||
if peer_process_id != gateway_pid {
|
||||
return Err(format!(
|
||||
"peer pid {peer_process_id} does not match expected gateway pid {gateway_pid}"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct AuthenticatedUnixIncoming {
|
||||
listener: UnixListener,
|
||||
expected_uid: u32,
|
||||
expected_peer_pid: Option<u32>,
|
||||
}
|
||||
|
||||
impl AuthenticatedUnixIncoming {
|
||||
fn new(listener: UnixListener, expected_peer_pid: Option<u32>) -> Self {
|
||||
Self {
|
||||
listener,
|
||||
expected_uid: current_euid(),
|
||||
expected_peer_pid,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Stream for AuthenticatedUnixIncoming {
|
||||
type Item = io::Result<UnixStream>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.get_mut();
|
||||
loop {
|
||||
match this.listener.poll_accept(cx) {
|
||||
Poll::Ready(Ok((stream, _addr))) => {
|
||||
let authorized = peer_credentials(&stream).and_then(|peer| {
|
||||
authorize_peer_credentials(peer, this.expected_uid, this.expected_peer_pid)
|
||||
});
|
||||
match authorized {
|
||||
Ok(()) => return Poll::Ready(Some(Ok(stream))),
|
||||
Err(err) => {
|
||||
tracing::warn!(
|
||||
error = %err,
|
||||
"rejected vm compute driver UDS client"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_vm_launch_config(args: &Args) -> std::result::Result<VmLaunchConfig, String> {
|
||||
let root_disk = args
|
||||
.vm_root_disk
|
||||
.clone()
|
||||
.ok_or_else(|| "--vm-root-disk is required in internal VM mode".to_string())?;
|
||||
let overlay_disk = args
|
||||
.vm_overlay_disk
|
||||
.clone()
|
||||
.ok_or_else(|| "--vm-overlay-disk is required in internal VM mode".to_string())?;
|
||||
let image_disk = args.vm_image_disk.clone();
|
||||
let exec_path = args
|
||||
.vm_exec
|
||||
.clone()
|
||||
.ok_or_else(|| "--vm-exec is required in internal VM mode".to_string())?;
|
||||
let console_output = args
|
||||
.vm_console_output
|
||||
.clone()
|
||||
.ok_or_else(|| "--vm-console-output is required in internal VM mode".to_string())?;
|
||||
|
||||
let backend = match args.vm_backend.as_deref() {
|
||||
Some("qemu") => VmBackend::Qemu,
|
||||
Some("libkrun") | None => VmBackend::Libkrun,
|
||||
Some(other) => return Err(format!("unknown VM backend: {other}")),
|
||||
};
|
||||
|
||||
Ok(VmLaunchConfig {
|
||||
root_disk,
|
||||
overlay_disk,
|
||||
image_disk,
|
||||
kernel_image: args.vm_kernel_image.clone(),
|
||||
vcpus: args.vm_vcpus,
|
||||
mem_mib: args.vm_mem_mib,
|
||||
exec_path,
|
||||
args: Vec::new(),
|
||||
env: args.vm_env.clone(),
|
||||
workdir: args.vm_workdir.clone(),
|
||||
log_level: args.vm_krun_log_level,
|
||||
console_output,
|
||||
backend,
|
||||
gpu_bdf: args.vm_gpu_bdf.clone(),
|
||||
tap_device: args.vm_tap_device.clone(),
|
||||
guest_ip: args.vm_guest_ip.clone(),
|
||||
host_ip: args.vm_host_ip.clone(),
|
||||
vsock_cid: args.vm_vsock_cid,
|
||||
guest_mac: args.vm_guest_mac.clone(),
|
||||
gateway_port: args.vm_gateway_port,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "macos")]
|
||||
fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> {
|
||||
use std::os::unix::process::CommandExt as _;
|
||||
|
||||
const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC";
|
||||
|
||||
if std::env::var_os(REEXEC_ENV).is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let runtime_dir = configured_runtime_dir().map_err(|err| miette::miette!("{err}"))?;
|
||||
let runtime_str = runtime_dir.to_string_lossy();
|
||||
let needs_reexec = std::env::var_os("DYLD_LIBRARY_PATH")
|
||||
.is_none_or(|value| !value.to_string_lossy().contains(runtime_str.as_ref()));
|
||||
if !needs_reexec {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mut dyld_paths = vec![runtime_dir.clone()];
|
||||
if let Some(existing) = std::env::var_os("DYLD_LIBRARY_PATH") {
|
||||
dyld_paths.extend(std::env::split_paths(&existing));
|
||||
}
|
||||
let joined = std::env::join_paths(&dyld_paths)
|
||||
.map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?;
|
||||
let exe = std::env::current_exe().into_diagnostic()?;
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
|
||||
// Use execvp() so the current process is *replaced* by the re-exec'd
|
||||
// binary — no wrapper process sits between the compute driver and
|
||||
// the actually-running VM launcher. That avoids two problems:
|
||||
// 1. An extra process level that survives SIGKILL of the driver
|
||||
// (the wrapper was reparenting the re-exec'd child to init).
|
||||
// 2. Signal forwarding: with a wrapper, a SIGTERM to the wrapper
|
||||
// doesn't reach the child unless we hand-roll forwarding.
|
||||
// After exec, the child inherits our PID and our procguard arming.
|
||||
let err = std::process::Command::new(exe)
|
||||
.args(&args)
|
||||
.env("DYLD_LIBRARY_PATH", &joined)
|
||||
.env(VM_RUNTIME_DIR_ENV, runtime_dir)
|
||||
.env(REEXEC_ENV, "1")
|
||||
.exec();
|
||||
// `exec()` only returns on failure.
|
||||
Err(miette::miette!("failed to re-exec with runtime env: {err}"))
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "macos"))]
|
||||
// Signature must match the macOS variant which can fail.
|
||||
#[allow(clippy::unnecessary_wraps)]
|
||||
fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
Args, ComputeDriverListenMode, PeerCredentials, authorize_peer_credentials,
|
||||
compute_driver_listen_mode,
|
||||
};
|
||||
use clap::Parser;
|
||||
use std::path::PathBuf;
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_accepts_matching_uid_and_pid() {
|
||||
authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1000,
|
||||
pid: Some(42),
|
||||
},
|
||||
1000,
|
||||
Some(42),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_rejects_wrong_pid() {
|
||||
let err = authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1000,
|
||||
pid: Some(7),
|
||||
},
|
||||
1000,
|
||||
Some(42),
|
||||
)
|
||||
.expect_err("wrong pid should be rejected");
|
||||
assert!(err.contains("does not match expected gateway pid"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_rejects_wrong_uid() {
|
||||
let err = authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1001,
|
||||
pid: Some(42),
|
||||
},
|
||||
1000,
|
||||
Some(42),
|
||||
)
|
||||
.expect_err("wrong uid should be rejected");
|
||||
assert!(err.contains("does not match current euid"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_rejects_missing_pid_when_expected() {
|
||||
let err = authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1000,
|
||||
pid: None,
|
||||
},
|
||||
1000,
|
||||
Some(42),
|
||||
)
|
||||
.expect_err("missing pid should be rejected");
|
||||
assert!(err.contains("peer pid is unavailable"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn peer_authorization_accepts_matching_uid_without_expected_pid() {
|
||||
authorize_peer_credentials(
|
||||
PeerCredentials {
|
||||
uid: 1000,
|
||||
pid: None,
|
||||
},
|
||||
1000,
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_rejects_default_tcp() {
|
||||
let args = Args::parse_from(["openshell-driver-vm"]);
|
||||
let err = compute_driver_listen_mode(&args).expect_err("default TCP should be disabled");
|
||||
assert!(err.contains("--bind-socket is required"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_gateway_otlp_endpoint() {
|
||||
let args = Args::try_parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--otlp-endpoint",
|
||||
"http://127.0.0.1:4317",
|
||||
]);
|
||||
assert!(
|
||||
args.is_ok(),
|
||||
"VM driver should accept the gateway OTLP endpoint"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_rejects_bind_address_without_tcp_opt_in() {
|
||||
let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]);
|
||||
let err =
|
||||
compute_driver_listen_mode(&args).expect_err("TCP bind should require explicit opt-in");
|
||||
assert!(err.contains("--allow-unauthenticated-tcp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_requires_bind_address_with_tcp_opt_in() {
|
||||
let args = Args::parse_from(["openshell-driver-vm", "--allow-unauthenticated-tcp"]);
|
||||
let err =
|
||||
compute_driver_listen_mode(&args).expect_err("TCP opt-in should require an address");
|
||||
assert!(err.contains("--bind-address is required"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_accepts_explicit_unauthenticated_tcp() {
|
||||
let args = Args::parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--allow-unauthenticated-tcp",
|
||||
"--bind-address",
|
||||
"127.0.0.1:50061",
|
||||
]);
|
||||
assert_eq!(
|
||||
compute_driver_listen_mode(&args).unwrap(),
|
||||
ComputeDriverListenMode::Tcp("127.0.0.1:50061".parse().unwrap())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_requires_expected_peer_pid_for_uds() {
|
||||
let args = Args::parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--bind-socket",
|
||||
"/tmp/compute-driver.sock",
|
||||
]);
|
||||
let err = compute_driver_listen_mode(&args)
|
||||
.expect_err("UDS should require gateway peer pid by default");
|
||||
assert!(err.contains("--expected-peer-pid is required"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_accepts_uds_with_expected_peer_pid() {
|
||||
let args = Args::parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--bind-socket",
|
||||
"/tmp/compute-driver.sock",
|
||||
"--expected-peer-pid",
|
||||
"42",
|
||||
]);
|
||||
assert_eq!(
|
||||
compute_driver_listen_mode(&args).unwrap(),
|
||||
ComputeDriverListenMode::Unix {
|
||||
socket_path: PathBuf::from("/tmp/compute-driver.sock"),
|
||||
expected_peer_pid: Some(42),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn listen_mode_accepts_explicit_same_uid_uds_dev_mode() {
|
||||
let args = Args::parse_from([
|
||||
"openshell-driver-vm",
|
||||
"--bind-socket",
|
||||
"/tmp/compute-driver.sock",
|
||||
"--allow-same-uid-peer",
|
||||
]);
|
||||
assert_eq!(
|
||||
compute_driver_listen_mode(&args).unwrap(),
|
||||
ComputeDriverListenMode::Unix {
|
||||
socket_path: PathBuf::from("/tmp/compute-driver.sock"),
|
||||
expected_peer_pid: None,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
fn main() {
|
||||
eprintln!("openshell-driver-vm is unsupported on Windows");
|
||||
std::process::exit(1);
|
||||
}
|
||||
@@ -26,11 +26,11 @@ use policy::parse_policy;
|
||||
use queries::run_all_queries;
|
||||
use report::{render_compact, render_report};
|
||||
|
||||
/// Run the prover end-to-end and return an exit code.
|
||||
/// Run the prover end-to-end and return a result containing an exit code.
|
||||
///
|
||||
/// - `0` — pass (no critical/high findings, or all accepted)
|
||||
/// - `1` — fail (critical or high findings present)
|
||||
/// - `2` — input error
|
||||
/// - `Ok(0)` — pass (no findings, or all accepted)
|
||||
/// - `Ok(1)` — fail (one or more unaccepted findings present)
|
||||
/// - `Err(_)` — input or registry loading error
|
||||
///
|
||||
/// Binary and API capability registries are embedded at compile time.
|
||||
/// Pass `registry_dir` to override with a custom filesystem registry.
|
||||
@@ -105,10 +105,11 @@ mod tests {
|
||||
fn test_filesystem_policy() {
|
||||
let path = testdata_dir().join("policy.yaml");
|
||||
let model = parse_policy(&path).expect("failed to parse policy");
|
||||
let readable = model.filesystem_policy.readable_paths();
|
||||
let readable = model.filesystem_policy.readable_paths(None);
|
||||
assert!(readable.contains(&"/usr".to_owned()));
|
||||
assert!(readable.contains(&"/sandbox".to_owned()));
|
||||
assert!(readable.contains(&"/tmp".to_owned()));
|
||||
assert!(readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned()));
|
||||
}
|
||||
|
||||
// 3. Workdir NOT included by default (matches runtime behavior).
|
||||
@@ -121,8 +122,9 @@ filesystem_policy:
|
||||
- /usr
|
||||
";
|
||||
let model = policy::parse_policy_str(yaml).expect("parse");
|
||||
let readable = model.filesystem_policy.readable_paths();
|
||||
let readable = model.filesystem_policy.readable_paths(None);
|
||||
assert!(!readable.contains(&"/sandbox".to_owned()));
|
||||
assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned()));
|
||||
}
|
||||
|
||||
// 4. Workdir excluded when include_workdir: false.
|
||||
@@ -136,8 +138,9 @@ filesystem_policy:
|
||||
- /usr
|
||||
";
|
||||
let model = policy::parse_policy_str(yaml).expect("parse");
|
||||
let readable = model.filesystem_policy.readable_paths();
|
||||
let readable = model.filesystem_policy.readable_paths(None);
|
||||
assert!(!readable.contains(&"/sandbox".to_owned()));
|
||||
assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned()));
|
||||
}
|
||||
|
||||
// 5. No duplicate when workdir already in read_write.
|
||||
@@ -152,12 +155,30 @@ filesystem_policy:
|
||||
- /tmp
|
||||
";
|
||||
let model = policy::parse_policy_str(yaml).expect("parse");
|
||||
let readable = model.filesystem_policy.readable_paths();
|
||||
let readable = model.filesystem_policy.readable_paths(Some("/sandbox"));
|
||||
let sandbox_count = readable.iter().filter(|p| *p == "/sandbox").count();
|
||||
assert_eq!(sandbox_count, 1);
|
||||
}
|
||||
|
||||
// 6. End-to-end: testdata policy with a github credential in scope and a
|
||||
// 6. A resolved non-default workdir does not replace an explicit path.
|
||||
#[test]
|
||||
fn test_include_workdir_preserves_explicit_sandbox_path() {
|
||||
let yaml = r"
|
||||
version: 1
|
||||
filesystem_policy:
|
||||
include_workdir: true
|
||||
read_write:
|
||||
- /sandbox
|
||||
";
|
||||
let model = policy::parse_policy_str(yaml).expect("parse");
|
||||
let readable = model
|
||||
.filesystem_policy
|
||||
.readable_paths(Some("/workspace/project"));
|
||||
assert!(readable.contains(&"/sandbox".to_owned()));
|
||||
assert!(readable.contains(&"/workspace/project".to_owned()));
|
||||
}
|
||||
|
||||
// 7. End-to-end: testdata policy with a github credential in scope and a
|
||||
// bypass-L7 binary (git) emits an `l7_bypass_credentialed` finding.
|
||||
// The prover output is categorical, not severity-graded.
|
||||
#[test]
|
||||
|
||||
@@ -1,314 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Formal policy verification for `OpenShell` sandboxes.
|
||||
//!
|
||||
//! Encodes sandbox policies, binary capabilities, and credential scopes as Z3
|
||||
//! SMT constraints, then checks reachability queries to detect data exfiltration
|
||||
//! paths and write-bypass violations.
|
||||
|
||||
pub mod accepted_risks;
|
||||
pub mod credentials;
|
||||
pub mod finding;
|
||||
pub mod model;
|
||||
pub mod policy;
|
||||
pub mod queries;
|
||||
pub mod registry;
|
||||
pub mod report;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use miette::Result;
|
||||
|
||||
use accepted_risks::{apply_accepted_risks, load_accepted_risks};
|
||||
use model::build_model;
|
||||
use policy::parse_policy;
|
||||
use queries::run_all_queries;
|
||||
use report::{render_compact, render_report};
|
||||
|
||||
/// Run the prover end-to-end and return a result containing an exit code.
|
||||
///
|
||||
/// - `Ok(0)` — pass (no findings, or all accepted)
|
||||
/// - `Ok(1)` — fail (one or more unaccepted findings present)
|
||||
/// - `Err(_)` — input or registry loading error
|
||||
///
|
||||
/// Binary and API capability registries are embedded at compile time.
|
||||
/// Pass `registry_dir` to override with a custom filesystem registry.
|
||||
pub fn prove(
|
||||
policy_path: &str,
|
||||
credentials_path: &str,
|
||||
registry_dir: Option<&str>,
|
||||
accepted_risks_path: Option<&str>,
|
||||
compact: bool,
|
||||
) -> Result<i32> {
|
||||
let policy = parse_policy(Path::new(policy_path))?;
|
||||
|
||||
let (credential_set, binary_registry) = match registry_dir {
|
||||
Some(dir) => {
|
||||
let dir = Path::new(dir);
|
||||
(
|
||||
credentials::load_credential_set_from_dir(Path::new(credentials_path), dir)?,
|
||||
registry::load_binary_registry_from_dir(dir)?,
|
||||
)
|
||||
}
|
||||
None => (
|
||||
credentials::load_credential_set_embedded(Path::new(credentials_path))?,
|
||||
registry::load_embedded_binary_registry()?,
|
||||
),
|
||||
};
|
||||
|
||||
let z3_model = build_model(policy, credential_set, binary_registry);
|
||||
let mut findings = run_all_queries(&z3_model);
|
||||
|
||||
if let Some(ar_path) = accepted_risks_path {
|
||||
let accepted = load_accepted_risks(Path::new(ar_path))?;
|
||||
findings = apply_accepted_risks(findings, &accepted);
|
||||
}
|
||||
|
||||
let exit_code = if compact {
|
||||
render_compact(&findings, policy_path, credentials_path)
|
||||
} else {
|
||||
render_report(&findings, policy_path, credentials_path)
|
||||
};
|
||||
|
||||
Ok(exit_code)
|
||||
}
|
||||
|
||||
// ===========================================================================
|
||||
// Tests
|
||||
// ===========================================================================
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn testdata_dir() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata")
|
||||
}
|
||||
|
||||
// 1. Parse testdata/policy.yaml, verify structure.
|
||||
#[test]
|
||||
fn test_parse_policy() {
|
||||
let path = testdata_dir().join("policy.yaml");
|
||||
let model = parse_policy(&path).expect("failed to parse policy");
|
||||
assert_eq!(model.version, 1);
|
||||
assert!(model.network_policies.contains_key("github_api"));
|
||||
let rule = &model.network_policies["github_api"];
|
||||
assert_eq!(rule.name, "github-api");
|
||||
assert_eq!(rule.endpoints.len(), 2);
|
||||
assert!(rule.binaries.len() >= 4);
|
||||
}
|
||||
|
||||
// 2. Verify readable_paths.
|
||||
#[test]
|
||||
fn test_filesystem_policy() {
|
||||
let path = testdata_dir().join("policy.yaml");
|
||||
let model = parse_policy(&path).expect("failed to parse policy");
|
||||
let readable = model.filesystem_policy.readable_paths(None);
|
||||
assert!(readable.contains(&"/usr".to_owned()));
|
||||
assert!(readable.contains(&"/sandbox".to_owned()));
|
||||
assert!(readable.contains(&"/tmp".to_owned()));
|
||||
assert!(readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned()));
|
||||
}
|
||||
|
||||
// 3. Workdir NOT included by default (matches runtime behavior).
|
||||
#[test]
|
||||
fn test_include_workdir_default() {
|
||||
let yaml = r"
|
||||
version: 1
|
||||
filesystem_policy:
|
||||
read_only:
|
||||
- /usr
|
||||
";
|
||||
let model = policy::parse_policy_str(yaml).expect("parse");
|
||||
let readable = model.filesystem_policy.readable_paths(None);
|
||||
assert!(!readable.contains(&"/sandbox".to_owned()));
|
||||
assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned()));
|
||||
}
|
||||
|
||||
// 4. Workdir excluded when include_workdir: false.
|
||||
#[test]
|
||||
fn test_include_workdir_false() {
|
||||
let yaml = r"
|
||||
version: 1
|
||||
filesystem_policy:
|
||||
include_workdir: false
|
||||
read_only:
|
||||
- /usr
|
||||
";
|
||||
let model = policy::parse_policy_str(yaml).expect("parse");
|
||||
let readable = model.filesystem_policy.readable_paths(None);
|
||||
assert!(!readable.contains(&"/sandbox".to_owned()));
|
||||
assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned()));
|
||||
}
|
||||
|
||||
// 5. No duplicate when workdir already in read_write.
|
||||
#[test]
|
||||
fn test_include_workdir_no_duplicate() {
|
||||
let yaml = r"
|
||||
version: 1
|
||||
filesystem_policy:
|
||||
include_workdir: true
|
||||
read_write:
|
||||
- /sandbox
|
||||
- /tmp
|
||||
";
|
||||
let model = policy::parse_policy_str(yaml).expect("parse");
|
||||
let readable = model.filesystem_policy.readable_paths(Some("/sandbox"));
|
||||
let sandbox_count = readable.iter().filter(|p| *p == "/sandbox").count();
|
||||
assert_eq!(sandbox_count, 1);
|
||||
}
|
||||
|
||||
// 6. A resolved non-default workdir does not replace an explicit path.
|
||||
#[test]
|
||||
fn test_include_workdir_preserves_explicit_sandbox_path() {
|
||||
let yaml = r"
|
||||
version: 1
|
||||
filesystem_policy:
|
||||
include_workdir: true
|
||||
read_write:
|
||||
- /sandbox
|
||||
";
|
||||
let model = policy::parse_policy_str(yaml).expect("parse");
|
||||
let readable = model
|
||||
.filesystem_policy
|
||||
.readable_paths(Some("/workspace/project"));
|
||||
assert!(readable.contains(&"/sandbox".to_owned()));
|
||||
assert!(readable.contains(&"/workspace/project".to_owned()));
|
||||
}
|
||||
|
||||
// 7. End-to-end: testdata policy with a github credential in scope and a
|
||||
// bypass-L7 binary (git) emits an `l7_bypass_credentialed` finding.
|
||||
// The prover output is categorical, not severity-graded.
|
||||
#[test]
|
||||
fn test_findings_for_github_policy() {
|
||||
use finding::category;
|
||||
|
||||
let policy_path = testdata_dir().join("policy.yaml");
|
||||
let creds_path = testdata_dir().join("credentials.yaml");
|
||||
|
||||
let pol = parse_policy(&policy_path).expect("parse policy");
|
||||
let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds");
|
||||
let bin_reg = registry::load_embedded_binary_registry().expect("load registry");
|
||||
|
||||
let z3_model = build_model(pol, cred_set, bin_reg);
|
||||
let findings = run_all_queries(&z3_model);
|
||||
|
||||
let categories: std::collections::HashSet<&str> =
|
||||
findings.iter().map(|f| f.query.as_str()).collect();
|
||||
assert!(
|
||||
categories.contains(category::L7_BYPASS_CREDENTIALED),
|
||||
"expected l7_bypass_credentialed finding for bypass-L7 binary with credential in scope; \
|
||||
got categories: {categories:?}"
|
||||
);
|
||||
// Every emitted category must be one of the four v1 categories.
|
||||
let allowed: std::collections::HashSet<&str> = [
|
||||
category::LINK_LOCAL_REACH,
|
||||
category::L7_BYPASS_CREDENTIALED,
|
||||
category::CREDENTIAL_REACH_EXPANSION,
|
||||
category::CAPABILITY_EXPANSION,
|
||||
]
|
||||
.into_iter()
|
||||
.collect();
|
||||
for c in &categories {
|
||||
assert!(
|
||||
allowed.contains(*c),
|
||||
"unexpected category {c} emitted by the prover"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_wildcard_endpoint_covering_credential_host_emits_credential_reach() {
|
||||
use finding::{FindingPath, category};
|
||||
|
||||
let policy = policy::parse_policy_str(
|
||||
r#"
|
||||
version: 1
|
||||
network_policies:
|
||||
github_wildcard:
|
||||
name: github-wildcard
|
||||
endpoints:
|
||||
- host: "*.github.com"
|
||||
port: 443
|
||||
protocol: rest
|
||||
enforcement: enforce
|
||||
access: read-write
|
||||
binaries:
|
||||
- path: /usr/bin/curl
|
||||
"#,
|
||||
)
|
||||
.expect("parse policy");
|
||||
let cred_set =
|
||||
credentials::load_credential_set_embedded(&testdata_dir().join("credentials.yaml"))
|
||||
.expect("load creds");
|
||||
let bin_reg = registry::load_embedded_binary_registry().expect("load registry");
|
||||
|
||||
let z3_model = build_model(policy, cred_set, bin_reg);
|
||||
let findings = run_all_queries(&z3_model);
|
||||
|
||||
let reach = findings
|
||||
.iter()
|
||||
.find(|finding| finding.query == category::CREDENTIAL_REACH_EXPANSION)
|
||||
.expect("wildcard host covering api.github.com must be credentialed");
|
||||
assert!(reach.paths.iter().any(|path| matches!(
|
||||
path,
|
||||
FindingPath::Exfil(exfil)
|
||||
if exfil.endpoint_host == "*.github.com" && exfil.binary == "/usr/bin/curl"
|
||||
)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_known_metadata_hostname_emits_link_local_finding() {
|
||||
use finding::{FindingPath, category};
|
||||
|
||||
let policy = policy::parse_policy_str(
|
||||
r"
|
||||
version: 1
|
||||
network_policies:
|
||||
metadata:
|
||||
name: metadata
|
||||
endpoints:
|
||||
- host: metadata.google.internal
|
||||
port: 80
|
||||
binaries:
|
||||
- path: /usr/bin/curl
|
||||
",
|
||||
)
|
||||
.expect("parse policy");
|
||||
let bin_reg = registry::load_embedded_binary_registry().expect("load registry");
|
||||
|
||||
let z3_model = build_model(policy, credentials::CredentialSet::default(), bin_reg);
|
||||
let findings = run_all_queries(&z3_model);
|
||||
|
||||
let link_local = findings
|
||||
.iter()
|
||||
.find(|finding| finding.query == category::LINK_LOCAL_REACH)
|
||||
.expect("known metadata hostname must emit link-local/metadata finding");
|
||||
assert!(link_local.paths.iter().any(|path| matches!(
|
||||
path,
|
||||
FindingPath::Exfil(exfil)
|
||||
if exfil.endpoint_host == "metadata.google.internal"
|
||||
)));
|
||||
}
|
||||
|
||||
// 7. Empty policy produces no findings.
|
||||
#[test]
|
||||
fn test_empty_policy_no_findings() {
|
||||
let policy_path = testdata_dir().join("empty-policy.yaml");
|
||||
let creds_path = testdata_dir().join("credentials.yaml");
|
||||
|
||||
let pol = parse_policy(&policy_path).expect("parse policy");
|
||||
let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds");
|
||||
let bin_reg = registry::load_embedded_binary_registry().expect("load registry");
|
||||
|
||||
let z3_model = build_model(pol, cred_set, bin_reg);
|
||||
let findings = run_all_queries(&z3_model);
|
||||
|
||||
assert!(
|
||||
findings.is_empty(),
|
||||
"deny-all policy should produce no findings, got: {findings:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
|
||||
[package]
|
||||
name = "openshell-sandbox"
|
||||
autobins = false
|
||||
description = "OpenShell process sandbox and monitor"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
@@ -10,9 +11,12 @@ rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
path = "src/lib_entry.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "openshell-sandbox"
|
||||
path = "src/main.rs"
|
||||
path = "src/main_entry.rs"
|
||||
|
||||
[target.'cfg(not(target_os = "windows"))'.dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("lib.rs");
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
include!("lib_win.rs");
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,6 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
pub fn windows_unsupported() -> &'static str {
|
||||
"openshell-sandbox supervisor is not available on Windows"
|
||||
}
|
||||
@@ -1,11 +1,822 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
fn main() {
|
||||
eprintln!("openshell-sandbox supervisor is not available on Windows");
|
||||
std::process::exit(1);
|
||||
//! `OpenShell` Sandbox - process sandbox and monitor.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
use clap::Parser;
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer};
|
||||
use tracing::{info, warn};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::filter::LevelFilter;
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
|
||||
use openshell_sandbox::run_sandbox;
|
||||
|
||||
/// Subcommand name used to self-copy the supervisor binary into a shared volume.
|
||||
///
|
||||
/// Init containers invoke the binary directly instead of relying on `sh`/`cp`
|
||||
/// to copy the binary out. Invoking the binary itself with this argument
|
||||
/// performs the copy in pure Rust.
|
||||
const COPY_SELF_SUBCOMMAND: &str = "copy-self";
|
||||
|
||||
/// Subcommand for one-shot debug RPCs from inside a sandbox container.
|
||||
///
|
||||
/// Reads the same token sources as the supervisor (env, file, K8s SA
|
||||
/// bootstrap) and issues a single gRPC call against the gateway. Useful
|
||||
/// for end-to-end verification: e.g. `docker exec` into a sandbox, then
|
||||
/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id <other>`
|
||||
/// to confirm the cross-sandbox IDOR guard fires.
|
||||
const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc";
|
||||
const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace";
|
||||
|
||||
/// Default `--mode` value: run both supervisor leaves in a single binary.
|
||||
const DEFAULT_MODE: &str = "network,process";
|
||||
const SIDECAR_STATE_DIR: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT;
|
||||
const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR;
|
||||
#[cfg(target_os = "linux")]
|
||||
const CLIENT_TLS_DIR: &str = openshell_core::container_paths::CLIENT_TLS_DIR;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client";
|
||||
#[cfg(target_os = "linux")]
|
||||
const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"];
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_STATE_DIR_MODE: u32 = 0o2775;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_TLS_DIR_MODE: u32 = 0o755;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400;
|
||||
|
||||
/// Which supervisor leaves are enabled in this process.
|
||||
///
|
||||
/// Parsed from a comma-separated `--mode` value, e.g. `network`,
|
||||
/// `process`, or `network,process`. `network-init` is a one-shot setup mode
|
||||
/// used by the Kubernetes sidecar topology and cannot be combined with other
|
||||
/// mode components. At least one must be set.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct Mode {
|
||||
network: bool,
|
||||
process: bool,
|
||||
network_init: bool,
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("main_unix.rs");
|
||||
impl std::str::FromStr for Mode {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
let mut mode = Self {
|
||||
network: false,
|
||||
process: false,
|
||||
network_init: false,
|
||||
};
|
||||
for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) {
|
||||
match part {
|
||||
"network" => mode.network = true,
|
||||
"process" => mode.process = true,
|
||||
"network-init" => mode.network_init = true,
|
||||
other => {
|
||||
return Err(format!(
|
||||
"unknown mode component '{other}' (expected 'network', 'process', or 'network-init')"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
if mode.network_init && (mode.network || mode.process) {
|
||||
return Err("--mode=network-init cannot be combined with other components".into());
|
||||
}
|
||||
if !mode.network && !mode.process && !mode.network_init {
|
||||
return Err(
|
||||
"--mode must enable at least one of: network, process, network-init".into(),
|
||||
);
|
||||
}
|
||||
Ok(mode)
|
||||
}
|
||||
}
|
||||
|
||||
/// `OpenShell` Sandbox - process isolation and monitoring.
|
||||
// CLI flags are naturally boolean switches; grouping them into structs would
|
||||
// only obscure the clap definition.
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "openshell-sandbox")]
|
||||
#[command(version = openshell_core::VERSION)]
|
||||
#[command(about = "Process sandbox and monitor", long_about = None)]
|
||||
struct Args {
|
||||
/// Command to execute in the sandbox.
|
||||
/// Can also be provided via `OPENSHELL_SANDBOX_COMMAND` environment variable.
|
||||
/// Defaults to `/bin/bash` if neither is provided.
|
||||
#[arg(trailing_var_arg = true)]
|
||||
command: Vec<String>,
|
||||
|
||||
/// Working directory for the sandboxed process.
|
||||
#[arg(long, short)]
|
||||
workdir: Option<String>,
|
||||
|
||||
/// Timeout in seconds (0 = no timeout).
|
||||
#[arg(long, short, default_value = "0")]
|
||||
timeout: u64,
|
||||
|
||||
/// Run in interactive mode (inherit process group for terminal control).
|
||||
#[arg(long, short = 'i')]
|
||||
interactive: bool,
|
||||
|
||||
/// Sandbox ID for fetching policy via gRPC from `OpenShell` server.
|
||||
/// Requires --openshell-endpoint to be set.
|
||||
#[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)]
|
||||
sandbox_id: Option<String>,
|
||||
|
||||
/// Sandbox (used for policy sync when the sandbox discovers policy
|
||||
/// from disk or falls back to the restrictive default).
|
||||
#[arg(long, env = openshell_core::sandbox_env::SANDBOX)]
|
||||
sandbox: Option<String>,
|
||||
|
||||
/// `OpenShell` server gRPC endpoint for fetching policy.
|
||||
/// Required when using --sandbox-id.
|
||||
#[arg(long, env = openshell_core::sandbox_env::ENDPOINT)]
|
||||
openshell_endpoint: Option<String>,
|
||||
|
||||
/// Path to Rego policy file for OPA-based network access control.
|
||||
/// Requires --policy-data to also be set.
|
||||
#[arg(long, env = "OPENSHELL_POLICY_RULES")]
|
||||
policy_rules: Option<String>,
|
||||
|
||||
/// Path to YAML data file containing network policies and sandbox config.
|
||||
/// Requires --policy-rules to also be set.
|
||||
#[arg(long, env = "OPENSHELL_POLICY_DATA")]
|
||||
policy_data: Option<String>,
|
||||
|
||||
/// Log level (trace, debug, info, warn, error).
|
||||
#[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)]
|
||||
log_level: String,
|
||||
|
||||
/// Unix socket the embedded SSH daemon binds. On Linux, a value beginning
|
||||
/// with `@` selects an abstract socket in the network namespace.
|
||||
/// The supervisor bridges `RelayStream` traffic from the gateway onto
|
||||
/// this socket; nothing else should connect to it.
|
||||
#[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)]
|
||||
ssh_socket_path: Option<String>,
|
||||
|
||||
/// Path to YAML inference routes for standalone routing.
|
||||
/// When set, inference routes are loaded from this file instead of
|
||||
/// fetching a bundle from the gateway.
|
||||
#[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")]
|
||||
inference_routes: Option<String>,
|
||||
|
||||
/// Enable health check endpoint.
|
||||
#[arg(long)]
|
||||
health_check: bool,
|
||||
|
||||
/// Port for health check endpoint.
|
||||
#[arg(long, default_value = "8080")]
|
||||
health_port: u16,
|
||||
|
||||
/// Which supervisor components to run. Comma-separated list of
|
||||
/// "network" and/or "process". Defaults to both (single-binary
|
||||
/// topology). Use --mode=network for a network-only sidecar, or
|
||||
/// --mode=process for a process-only supervisor when network
|
||||
/// enforcement runs in another pod. Use --mode=network-init only in
|
||||
/// the Kubernetes init container that prepares sidecar nftables.
|
||||
#[arg(long, default_value = DEFAULT_MODE)]
|
||||
mode: Mode,
|
||||
|
||||
/// UID that the long-running Kubernetes network sidecar will run as.
|
||||
/// `--mode=network-init` installs nftables rules that exempt this UID.
|
||||
#[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)]
|
||||
proxy_uid: u32,
|
||||
|
||||
/// GID assigned to shared sidecar state directories. Defaults to
|
||||
/// `--proxy-uid` when omitted.
|
||||
#[arg(long, env = "OPENSHELL_PROXY_GID")]
|
||||
proxy_gid: Option<u32>,
|
||||
|
||||
/// Shared state directory between the network init container and sidecar.
|
||||
#[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)]
|
||||
sidecar_state_dir: String,
|
||||
|
||||
/// Shared TLS work directory between the network init container and sidecar.
|
||||
#[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)]
|
||||
sidecar_tls_dir: String,
|
||||
|
||||
// Corporate upstream proxy. Operator-owned egress boundary: accepted
|
||||
// only as command-line arguments (no `env =`), because the driver
|
||||
// controls the supervisor's argv while a sandbox image could bake
|
||||
// matching `ENV` values.
|
||||
/// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress.
|
||||
#[arg(long)]
|
||||
upstream_proxy: Option<String>,
|
||||
|
||||
/// Comma-separated `NO_PROXY` list for the corporate proxy.
|
||||
#[arg(long)]
|
||||
upstream_no_proxy: Option<String>,
|
||||
|
||||
/// Path to the root-only file holding corporate proxy credentials (`user:pass`).
|
||||
#[arg(long)]
|
||||
upstream_proxy_auth_file: Option<String>,
|
||||
|
||||
/// Acknowledge that proxy credentials travel as cleartext Basic auth over
|
||||
/// the plain-TCP connection to the `http://` proxy.
|
||||
#[arg(long)]
|
||||
upstream_proxy_auth_allow_insecure: bool,
|
||||
|
||||
/// Send the destination hostname in CONNECT instead of a validated IP
|
||||
/// (for proxies whose ACLs filter on hostnames).
|
||||
#[arg(long)]
|
||||
upstream_proxy_connect_by_hostname: bool,
|
||||
}
|
||||
|
||||
/// Internal one-shot command used by the privileged supervisor to validate an
|
||||
/// image-provided workdir as the final sandbox identity.
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "validate-workspace", hide = true)]
|
||||
struct ValidateWorkspaceArgs {
|
||||
#[arg(long)]
|
||||
workdir: String,
|
||||
#[arg(long)]
|
||||
expected_uid: u32,
|
||||
#[arg(long)]
|
||||
expected_gid: u32,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn validate_workspace(args: &[String]) -> Result<()> {
|
||||
let args = ValidateWorkspaceArgs::try_parse_from(
|
||||
std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()),
|
||||
)
|
||||
.into_diagnostic()?;
|
||||
let actual = (
|
||||
nix::unistd::geteuid().as_raw(),
|
||||
nix::unistd::getegid().as_raw(),
|
||||
);
|
||||
if actual != (args.expected_uid, args.expected_gid) {
|
||||
return Err(miette::miette!(
|
||||
"workspace validator privilege drop failed: expected {}:{}, got {}:{}",
|
||||
args.expected_uid,
|
||||
args.expected_gid,
|
||||
actual.0,
|
||||
actual.1
|
||||
));
|
||||
}
|
||||
openshell_supervisor_process::process::validate_oci_workspace_as_effective_identity(Path::new(
|
||||
&args.workdir,
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn validate_workspace(_args: &[String]) -> Result<()> {
|
||||
Err(miette::miette!(
|
||||
"workspace validation is only supported on Unix"
|
||||
))
|
||||
}
|
||||
|
||||
/// Copy the running executable to `dest`, creating parent directories as
|
||||
/// needed and ensuring the result is executable (mode `0755`).
|
||||
///
|
||||
/// If `dest` already exists as a directory, the binary is placed inside it
|
||||
/// using the source executable's file name. This mirrors `cp` semantics so
|
||||
/// callers can pass either a full target path or a directory.
|
||||
fn copy_self(dest: &str) -> Result<()> {
|
||||
let exe = std::env::current_exe().into_diagnostic()?;
|
||||
|
||||
let dest_path = Path::new(dest);
|
||||
let final_path = if dest_path.is_dir() {
|
||||
let file_name = exe
|
||||
.file_name()
|
||||
.ok_or_else(|| miette::miette!("current_exe has no file name: {}", exe.display()))?;
|
||||
dest_path.join(file_name)
|
||||
} else {
|
||||
dest_path.to_path_buf()
|
||||
};
|
||||
|
||||
if let Some(parent) = final_path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
std::fs::create_dir_all(parent).into_diagnostic()?;
|
||||
}
|
||||
|
||||
std::fs::copy(&exe, &final_path).into_diagnostic()?;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mut perms = std::fs::metadata(&final_path)
|
||||
.into_diagnostic()?
|
||||
.permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&final_path, perms).into_diagnostic()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> {
|
||||
use miette::Context as _;
|
||||
use nix::unistd::{Gid, Uid, chown};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
std::fs::create_dir_all(path)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?;
|
||||
let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions();
|
||||
perms.set_mode(mode);
|
||||
std::fs::set_permissions(path, perms)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?;
|
||||
chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid)))
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to chown sidecar directory {} to {uid}:{gid}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> {
|
||||
use miette::Context as _;
|
||||
use nix::unistd::{Gid, Uid, chown};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let uid = Uid::current();
|
||||
let gid = Gid::current();
|
||||
std::fs::create_dir_all(path)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?;
|
||||
chown(path, Some(uid), Some(gid))
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to chown sidecar directory {} to {}:{}",
|
||||
path.display(),
|
||||
uid.as_raw(),
|
||||
gid.as_raw()
|
||||
)
|
||||
})?;
|
||||
let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions();
|
||||
perms.set_mode(mode);
|
||||
std::fs::set_permissions(path, perms)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn copy_sidecar_client_tls_if_present(
|
||||
source_dir: &Path,
|
||||
sidecar_tls_dir: &Path,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
) -> Result<()> {
|
||||
use miette::Context as _;
|
||||
use nix::unistd::{Gid, Uid, chown};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
if !source_dir.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR);
|
||||
prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?;
|
||||
for file_name in CLIENT_TLS_FILES {
|
||||
let source = source_dir.join(file_name);
|
||||
if !source.exists() {
|
||||
return Err(miette::miette!(
|
||||
"client TLS source file is missing: {}",
|
||||
source.display()
|
||||
));
|
||||
}
|
||||
let dest = dest_dir.join(file_name);
|
||||
if dest.exists() {
|
||||
std::fs::remove_file(&dest)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!("failed to remove stale client TLS file {}", dest.display())
|
||||
})?;
|
||||
}
|
||||
std::fs::copy(&source, &dest)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to copy client TLS file {} to {}",
|
||||
source.display(),
|
||||
dest.display()
|
||||
)
|
||||
})?;
|
||||
let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions();
|
||||
perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE);
|
||||
std::fs::set_permissions(&dest, perms)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!("failed to chmod copied client TLS file {}", dest.display())
|
||||
})?;
|
||||
chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid)))
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to chown copied client TLS file {} to {uid}:{gid}",
|
||||
dest.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn run_network_init(
|
||||
proxy_user_id: u32,
|
||||
proxy_primary_group_id: u32,
|
||||
sidecar_state_dir: &str,
|
||||
sidecar_tls_dir: &str,
|
||||
) -> Result<()> {
|
||||
validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?;
|
||||
|
||||
let sidecar_state_dir = Path::new(sidecar_state_dir);
|
||||
let sidecar_tls_dir = Path::new(sidecar_tls_dir);
|
||||
prepare_sidecar_directory(
|
||||
sidecar_state_dir,
|
||||
proxy_user_id,
|
||||
proxy_primary_group_id,
|
||||
SIDECAR_STATE_DIR_MODE,
|
||||
)?;
|
||||
// The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the
|
||||
// TLS work directory owned by the init user until the client cert copy is
|
||||
// complete, then hand it to the long-running proxy UID.
|
||||
prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?;
|
||||
copy_sidecar_client_tls_if_present(
|
||||
Path::new(CLIENT_TLS_DIR),
|
||||
sidecar_tls_dir,
|
||||
proxy_user_id,
|
||||
proxy_primary_group_id,
|
||||
)?;
|
||||
prepare_sidecar_directory(
|
||||
sidecar_tls_dir,
|
||||
proxy_user_id,
|
||||
proxy_primary_group_id,
|
||||
SIDECAR_TLS_DIR_MODE,
|
||||
)?;
|
||||
openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> {
|
||||
if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID {
|
||||
return Err(miette::miette!(
|
||||
"--proxy-uid must be 0 or at least {}",
|
||||
openshell_policy::MIN_SANDBOX_UID
|
||||
));
|
||||
}
|
||||
if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID {
|
||||
return Err(miette::miette!(
|
||||
"--proxy-gid must be at least {}",
|
||||
openshell_policy::MIN_SANDBOX_UID
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn run_network_init(
|
||||
_proxy_uid: u32,
|
||||
_proxy_gid: u32,
|
||||
_sidecar_state_dir: &str,
|
||||
_sidecar_tls_dir: &str,
|
||||
) -> Result<()> {
|
||||
Err(miette::miette!(
|
||||
"--mode=network-init is only supported on Linux"
|
||||
))
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// Handle `copy-self <DEST>` before clap so it works without any of the
|
||||
// sandbox flags. Kubernetes init containers invoke this path to seed an
|
||||
// emptyDir volume that the agent container then executes from.
|
||||
let raw_args: Vec<String> = std::env::args().collect();
|
||||
if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) {
|
||||
let dest = raw_args.get(2).ok_or_else(|| {
|
||||
miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} <DEST>")
|
||||
})?;
|
||||
return copy_self(dest);
|
||||
}
|
||||
|
||||
// Handle `debug-rpc <subcommand> [args]` before clap. Uses a small
|
||||
// dedicated runtime so we don't pay the supervisor's full startup cost.
|
||||
if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.into_diagnostic()?;
|
||||
return runtime.block_on(async move {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?;
|
||||
std::process::exit(exit);
|
||||
});
|
||||
}
|
||||
if raw_args.get(1).map(String::as_str) == Some(VALIDATE_WORKSPACE_SUBCOMMAND) {
|
||||
return validate_workspace(&raw_args[2..]);
|
||||
}
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
if args.mode.network_init {
|
||||
let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid);
|
||||
return run_network_init(
|
||||
args.proxy_uid,
|
||||
proxy_gid,
|
||||
&args.sidecar_state_dir,
|
||||
&args.sidecar_tls_dir,
|
||||
);
|
||||
}
|
||||
|
||||
// Try to open a rolling log file; fall back to stderr-only logging if it fails
|
||||
// (e.g., /var/log is not writable in custom workload images).
|
||||
// Rotates daily, keeps the 3 most recent files to bound disk usage.
|
||||
let file_logging = tracing_appender::rolling::RollingFileAppender::builder()
|
||||
.rotation(tracing_appender::rolling::Rotation::DAILY)
|
||||
.filename_prefix("openshell")
|
||||
.filename_suffix("log")
|
||||
.max_log_files(3)
|
||||
.build("/var/log")
|
||||
.ok()
|
||||
.map(|roller| {
|
||||
let (writer, guard) = tracing_appender::non_blocking(roller);
|
||||
(writer, guard)
|
||||
});
|
||||
|
||||
let console_filter =
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level));
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.into_diagnostic()?;
|
||||
|
||||
let exit_code = runtime.block_on(async move {
|
||||
// Install rustls crypto provider before any TLS connections (including log push).
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
|
||||
// Set up optional log push layer (gRPC mode only).
|
||||
let log_push_state = if let (Some(sandbox_id), Some(endpoint)) =
|
||||
(&args.sandbox_id, &args.openshell_endpoint)
|
||||
{
|
||||
let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task(
|
||||
endpoint.clone(),
|
||||
sandbox_id.clone(),
|
||||
);
|
||||
let layer =
|
||||
openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx);
|
||||
Some((layer, handle))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone());
|
||||
let _log_push_handle = log_push_state.map(|(_, handle)| handle);
|
||||
|
||||
// Shared flag: the sandbox poll loop toggles this when the
|
||||
// `ocsf_json_enabled` setting changes. The JSONL layer checks it
|
||||
// on each event and short-circuits when false.
|
||||
let ocsf_enabled = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Keep guards alive for the entire process. When a guard is dropped the
|
||||
// non-blocking writer flushes remaining logs.
|
||||
let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging {
|
||||
let file_filter = EnvFilter::new("info");
|
||||
|
||||
// OCSF JSONL file: rolling appender matching the main log file
|
||||
// (daily rotation, 3 files max). Created eagerly but gated by the
|
||||
// enabled flag — no JSONL is written until ocsf_json_enabled is set.
|
||||
let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder()
|
||||
.rotation(tracing_appender::rolling::Rotation::DAILY)
|
||||
.filename_prefix("openshell-ocsf")
|
||||
.filename_suffix("log")
|
||||
.max_log_files(3)
|
||||
.build("/var/log")
|
||||
.ok()
|
||||
.map(|roller| {
|
||||
let (writer, guard) = tracing_appender::non_blocking(roller);
|
||||
let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone());
|
||||
(layer, guard)
|
||||
});
|
||||
let (jsonl_layer, jsonl_guard) = match jsonl_logging {
|
||||
Some((layer, guard)) => (Some(layer), Some(guard)),
|
||||
None => (None, None),
|
||||
};
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(
|
||||
OcsfShorthandLayer::new(std::io::stderr())
|
||||
.with_non_ocsf(true)
|
||||
.with_filter(console_filter),
|
||||
)
|
||||
.with(
|
||||
OcsfShorthandLayer::new(file_writer)
|
||||
.with_non_ocsf(true)
|
||||
.with_filter(file_filter),
|
||||
)
|
||||
.with(jsonl_layer.with_filter(LevelFilter::INFO))
|
||||
.with(push_layer.clone())
|
||||
.init();
|
||||
(Some(file_guard), jsonl_guard)
|
||||
} else {
|
||||
tracing_subscriber::registry()
|
||||
.with(
|
||||
OcsfShorthandLayer::new(std::io::stderr())
|
||||
.with_non_ocsf(true)
|
||||
.with_filter(console_filter),
|
||||
)
|
||||
.with(push_layer)
|
||||
.init();
|
||||
// Log the warning after the subscriber is initialized
|
||||
warn!("Could not open /var/log for log rotation; using stderr-only logging");
|
||||
(None, None)
|
||||
};
|
||||
|
||||
// Get command - either from CLI args, environment variable, or default to /bin/bash
|
||||
let command = if !args.command.is_empty() {
|
||||
args.command
|
||||
} else if let Ok(c) = std::env::var(openshell_core::sandbox_env::SANDBOX_COMMAND) {
|
||||
// Simple shell-like splitting on whitespace
|
||||
c.split_whitespace().map(String::from).collect()
|
||||
} else {
|
||||
vec!["/bin/bash".to_string()]
|
||||
};
|
||||
|
||||
info!(command = ?command, "Starting sandbox");
|
||||
// Note: "Starting sandbox" stays as plain info!() since the OCSF context
|
||||
// is not yet initialized at this point (run_sandbox hasn't been called).
|
||||
// The shorthand layer will render it in fallback format.
|
||||
|
||||
let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs {
|
||||
https_proxy: args.upstream_proxy,
|
||||
no_proxy: args.upstream_no_proxy,
|
||||
proxy_auth_file: args.upstream_proxy_auth_file,
|
||||
proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure,
|
||||
proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname,
|
||||
};
|
||||
|
||||
run_sandbox(
|
||||
command,
|
||||
args.workdir,
|
||||
args.timeout,
|
||||
args.interactive,
|
||||
args.sandbox_id,
|
||||
args.sandbox,
|
||||
args.openshell_endpoint,
|
||||
args.policy_rules,
|
||||
args.policy_data,
|
||||
args.ssh_socket_path,
|
||||
args.health_check,
|
||||
args.health_port,
|
||||
args.inference_routes,
|
||||
ocsf_enabled,
|
||||
args.mode.network,
|
||||
args.mode.process,
|
||||
upstream_proxy_args,
|
||||
)
|
||||
.await
|
||||
})?;
|
||||
|
||||
std::process::exit(exit_code);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn workspace_validation_subcommand_uses_final_policy_identity() {
|
||||
let uid = nix::unistd::geteuid().as_raw();
|
||||
let gid = nix::unistd::getegid().as_raw();
|
||||
if uid < 1000 || gid < 1000 {
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir_in("/tmp").unwrap();
|
||||
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap();
|
||||
let root = dir.path().canonicalize().unwrap().join("workspace");
|
||||
std::fs::create_dir(&root).unwrap();
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
let args = vec![
|
||||
"--workdir".to_string(),
|
||||
root.display().to_string(),
|
||||
"--expected-uid".to_string(),
|
||||
uid.to_string(),
|
||||
"--expected-gid".to_string(),
|
||||
gid.to_string(),
|
||||
];
|
||||
|
||||
validate_workspace(&args).expect("current identity should retain workspace authority");
|
||||
}
|
||||
|
||||
/// Drives `copy_self`'s file-copy logic against an arbitrary source path
|
||||
/// so tests don't depend on `current_exe()`.
|
||||
fn copy_executable(src: &Path, dest: &Path) -> Result<()> {
|
||||
let final_path = if dest.is_dir() {
|
||||
dest.join(src.file_name().unwrap())
|
||||
} else {
|
||||
dest.to_path_buf()
|
||||
};
|
||||
if let Some(parent) = final_path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
std::fs::create_dir_all(parent).into_diagnostic()?;
|
||||
}
|
||||
std::fs::copy(src, &final_path).into_diagnostic()?;
|
||||
let mut perms = std::fs::metadata(&final_path)
|
||||
.into_diagnostic()?
|
||||
.permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&final_path, perms).into_diagnostic()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_self_writes_executable_at_target_path() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let src = tmp.path().join("source-bin");
|
||||
std::fs::write(&src, b"#!/bin/false\n").unwrap();
|
||||
|
||||
let dest = tmp.path().join("subdir/openshell-sandbox");
|
||||
copy_executable(&src, &dest).unwrap();
|
||||
|
||||
assert!(dest.exists(), "destination file should exist");
|
||||
let mode = std::fs::metadata(&dest).unwrap().permissions().mode();
|
||||
assert_eq!(mode & 0o777, 0o755, "destination must be 0755");
|
||||
let copied = std::fs::read(&dest).unwrap();
|
||||
assert_eq!(copied, b"#!/bin/false\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_self_into_existing_directory_uses_source_filename() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let src = tmp.path().join("openshell-sandbox");
|
||||
std::fs::write(&src, b"binary").unwrap();
|
||||
|
||||
let dest_dir = tmp.path().join("bin");
|
||||
std::fs::create_dir_all(&dest_dir).unwrap();
|
||||
|
||||
copy_executable(&src, &dest_dir).unwrap();
|
||||
|
||||
let final_path = dest_dir.join("openshell-sandbox");
|
||||
assert!(final_path.exists(), "binary should land inside dest dir");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_parses_network_init_standalone() {
|
||||
let mode = "network-init".parse::<Mode>().unwrap();
|
||||
assert!(mode.network_init);
|
||||
assert!(!mode.network);
|
||||
assert!(!mode.process);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_rejects_combined_network_init() {
|
||||
let err = "network-init,network".parse::<Mode>().unwrap_err();
|
||||
assert!(err.contains("cannot be combined"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_rejects_empty_value() {
|
||||
let err = "".parse::<Mode>().unwrap_err();
|
||||
assert!(err.contains("at least one"));
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() {
|
||||
assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755);
|
||||
assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700);
|
||||
assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750);
|
||||
assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() {
|
||||
validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap();
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn network_init_still_rejects_low_non_root_proxy_ids() {
|
||||
let uid_err =
|
||||
validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err();
|
||||
assert!(uid_err.to_string().contains("--proxy-uid"));
|
||||
|
||||
let gid_err = validate_network_init_ids(0, 999).unwrap_err();
|
||||
assert!(gid_err.to_string().contains("--proxy-gid"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
include!("main.rs");
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
include!("main_win.rs");
|
||||
@@ -1,822 +0,0 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! `OpenShell` Sandbox - process sandbox and monitor.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::AtomicBool;
|
||||
|
||||
use clap::Parser;
|
||||
use miette::{IntoDiagnostic, Result};
|
||||
use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer};
|
||||
use tracing::{info, warn};
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::filter::LevelFilter;
|
||||
use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
|
||||
use openshell_sandbox::run_sandbox;
|
||||
|
||||
/// Subcommand name used to self-copy the supervisor binary into a shared volume.
|
||||
///
|
||||
/// Init containers invoke the binary directly instead of relying on `sh`/`cp`
|
||||
/// to copy the binary out. Invoking the binary itself with this argument
|
||||
/// performs the copy in pure Rust.
|
||||
const COPY_SELF_SUBCOMMAND: &str = "copy-self";
|
||||
|
||||
/// Subcommand for one-shot debug RPCs from inside a sandbox container.
|
||||
///
|
||||
/// Reads the same token sources as the supervisor (env, file, K8s SA
|
||||
/// bootstrap) and issues a single gRPC call against the gateway. Useful
|
||||
/// for end-to-end verification: e.g. `docker exec` into a sandbox, then
|
||||
/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id <other>`
|
||||
/// to confirm the cross-sandbox IDOR guard fires.
|
||||
const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc";
|
||||
const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace";
|
||||
|
||||
/// Default `--mode` value: run both supervisor leaves in a single binary.
|
||||
const DEFAULT_MODE: &str = "network,process";
|
||||
const SIDECAR_STATE_DIR: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT;
|
||||
const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR;
|
||||
#[cfg(target_os = "linux")]
|
||||
const CLIENT_TLS_DIR: &str = openshell_core::container_paths::CLIENT_TLS_DIR;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client";
|
||||
#[cfg(target_os = "linux")]
|
||||
const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"];
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_STATE_DIR_MODE: u32 = 0o2775;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_TLS_DIR_MODE: u32 = 0o755;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750;
|
||||
#[cfg(target_os = "linux")]
|
||||
const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400;
|
||||
|
||||
/// Which supervisor leaves are enabled in this process.
|
||||
///
|
||||
/// Parsed from a comma-separated `--mode` value, e.g. `network`,
|
||||
/// `process`, or `network,process`. `network-init` is a one-shot setup mode
|
||||
/// used by the Kubernetes sidecar topology and cannot be combined with other
|
||||
/// mode components. At least one must be set.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct Mode {
|
||||
network: bool,
|
||||
process: bool,
|
||||
network_init: bool,
|
||||
}
|
||||
|
||||
impl std::str::FromStr for Mode {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
let mut mode = Self {
|
||||
network: false,
|
||||
process: false,
|
||||
network_init: false,
|
||||
};
|
||||
for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) {
|
||||
match part {
|
||||
"network" => mode.network = true,
|
||||
"process" => mode.process = true,
|
||||
"network-init" => mode.network_init = true,
|
||||
other => {
|
||||
return Err(format!(
|
||||
"unknown mode component '{other}' (expected 'network', 'process', or 'network-init')"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
if mode.network_init && (mode.network || mode.process) {
|
||||
return Err("--mode=network-init cannot be combined with other components".into());
|
||||
}
|
||||
if !mode.network && !mode.process && !mode.network_init {
|
||||
return Err(
|
||||
"--mode must enable at least one of: network, process, network-init".into(),
|
||||
);
|
||||
}
|
||||
Ok(mode)
|
||||
}
|
||||
}
|
||||
|
||||
/// `OpenShell` Sandbox - process isolation and monitoring.
|
||||
// CLI flags are naturally boolean switches; grouping them into structs would
|
||||
// only obscure the clap definition.
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "openshell-sandbox")]
|
||||
#[command(version = openshell_core::VERSION)]
|
||||
#[command(about = "Process sandbox and monitor", long_about = None)]
|
||||
struct Args {
|
||||
/// Command to execute in the sandbox.
|
||||
/// Can also be provided via `OPENSHELL_SANDBOX_COMMAND` environment variable.
|
||||
/// Defaults to `/bin/bash` if neither is provided.
|
||||
#[arg(trailing_var_arg = true)]
|
||||
command: Vec<String>,
|
||||
|
||||
/// Working directory for the sandboxed process.
|
||||
#[arg(long, short)]
|
||||
workdir: Option<String>,
|
||||
|
||||
/// Timeout in seconds (0 = no timeout).
|
||||
#[arg(long, short, default_value = "0")]
|
||||
timeout: u64,
|
||||
|
||||
/// Run in interactive mode (inherit process group for terminal control).
|
||||
#[arg(long, short = 'i')]
|
||||
interactive: bool,
|
||||
|
||||
/// Sandbox ID for fetching policy via gRPC from `OpenShell` server.
|
||||
/// Requires --openshell-endpoint to be set.
|
||||
#[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)]
|
||||
sandbox_id: Option<String>,
|
||||
|
||||
/// Sandbox (used for policy sync when the sandbox discovers policy
|
||||
/// from disk or falls back to the restrictive default).
|
||||
#[arg(long, env = openshell_core::sandbox_env::SANDBOX)]
|
||||
sandbox: Option<String>,
|
||||
|
||||
/// `OpenShell` server gRPC endpoint for fetching policy.
|
||||
/// Required when using --sandbox-id.
|
||||
#[arg(long, env = openshell_core::sandbox_env::ENDPOINT)]
|
||||
openshell_endpoint: Option<String>,
|
||||
|
||||
/// Path to Rego policy file for OPA-based network access control.
|
||||
/// Requires --policy-data to also be set.
|
||||
#[arg(long, env = "OPENSHELL_POLICY_RULES")]
|
||||
policy_rules: Option<String>,
|
||||
|
||||
/// Path to YAML data file containing network policies and sandbox config.
|
||||
/// Requires --policy-rules to also be set.
|
||||
#[arg(long, env = "OPENSHELL_POLICY_DATA")]
|
||||
policy_data: Option<String>,
|
||||
|
||||
/// Log level (trace, debug, info, warn, error).
|
||||
#[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)]
|
||||
log_level: String,
|
||||
|
||||
/// Unix socket the embedded SSH daemon binds. On Linux, a value beginning
|
||||
/// with `@` selects an abstract socket in the network namespace.
|
||||
/// The supervisor bridges `RelayStream` traffic from the gateway onto
|
||||
/// this socket; nothing else should connect to it.
|
||||
#[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)]
|
||||
ssh_socket_path: Option<String>,
|
||||
|
||||
/// Path to YAML inference routes for standalone routing.
|
||||
/// When set, inference routes are loaded from this file instead of
|
||||
/// fetching a bundle from the gateway.
|
||||
#[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")]
|
||||
inference_routes: Option<String>,
|
||||
|
||||
/// Enable health check endpoint.
|
||||
#[arg(long)]
|
||||
health_check: bool,
|
||||
|
||||
/// Port for health check endpoint.
|
||||
#[arg(long, default_value = "8080")]
|
||||
health_port: u16,
|
||||
|
||||
/// Which supervisor components to run. Comma-separated list of
|
||||
/// "network" and/or "process". Defaults to both (single-binary
|
||||
/// topology). Use --mode=network for a network-only sidecar, or
|
||||
/// --mode=process for a process-only supervisor when network
|
||||
/// enforcement runs in another pod. Use --mode=network-init only in
|
||||
/// the Kubernetes init container that prepares sidecar nftables.
|
||||
#[arg(long, default_value = DEFAULT_MODE)]
|
||||
mode: Mode,
|
||||
|
||||
/// UID that the long-running Kubernetes network sidecar will run as.
|
||||
/// `--mode=network-init` installs nftables rules that exempt this UID.
|
||||
#[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)]
|
||||
proxy_uid: u32,
|
||||
|
||||
/// GID assigned to shared sidecar state directories. Defaults to
|
||||
/// `--proxy-uid` when omitted.
|
||||
#[arg(long, env = "OPENSHELL_PROXY_GID")]
|
||||
proxy_gid: Option<u32>,
|
||||
|
||||
/// Shared state directory between the network init container and sidecar.
|
||||
#[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)]
|
||||
sidecar_state_dir: String,
|
||||
|
||||
/// Shared TLS work directory between the network init container and sidecar.
|
||||
#[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)]
|
||||
sidecar_tls_dir: String,
|
||||
|
||||
// Corporate upstream proxy. Operator-owned egress boundary: accepted
|
||||
// only as command-line arguments (no `env =`), because the driver
|
||||
// controls the supervisor's argv while a sandbox image could bake
|
||||
// matching `ENV` values.
|
||||
/// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress.
|
||||
#[arg(long)]
|
||||
upstream_proxy: Option<String>,
|
||||
|
||||
/// Comma-separated `NO_PROXY` list for the corporate proxy.
|
||||
#[arg(long)]
|
||||
upstream_no_proxy: Option<String>,
|
||||
|
||||
/// Path to the root-only file holding corporate proxy credentials (`user:pass`).
|
||||
#[arg(long)]
|
||||
upstream_proxy_auth_file: Option<String>,
|
||||
|
||||
/// Acknowledge that proxy credentials travel as cleartext Basic auth over
|
||||
/// the plain-TCP connection to the `http://` proxy.
|
||||
#[arg(long)]
|
||||
upstream_proxy_auth_allow_insecure: bool,
|
||||
|
||||
/// Send the destination hostname in CONNECT instead of a validated IP
|
||||
/// (for proxies whose ACLs filter on hostnames).
|
||||
#[arg(long)]
|
||||
upstream_proxy_connect_by_hostname: bool,
|
||||
}
|
||||
|
||||
/// Internal one-shot command used by the privileged supervisor to validate an
|
||||
/// image-provided workdir as the final sandbox identity.
|
||||
#[derive(Parser, Debug)]
|
||||
#[command(name = "validate-workspace", hide = true)]
|
||||
struct ValidateWorkspaceArgs {
|
||||
#[arg(long)]
|
||||
workdir: String,
|
||||
#[arg(long)]
|
||||
expected_uid: u32,
|
||||
#[arg(long)]
|
||||
expected_gid: u32,
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn validate_workspace(args: &[String]) -> Result<()> {
|
||||
let args = ValidateWorkspaceArgs::try_parse_from(
|
||||
std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()),
|
||||
)
|
||||
.into_diagnostic()?;
|
||||
let actual = (
|
||||
nix::unistd::geteuid().as_raw(),
|
||||
nix::unistd::getegid().as_raw(),
|
||||
);
|
||||
if actual != (args.expected_uid, args.expected_gid) {
|
||||
return Err(miette::miette!(
|
||||
"workspace validator privilege drop failed: expected {}:{}, got {}:{}",
|
||||
args.expected_uid,
|
||||
args.expected_gid,
|
||||
actual.0,
|
||||
actual.1
|
||||
));
|
||||
}
|
||||
openshell_supervisor_process::process::validate_oci_workspace_as_effective_identity(Path::new(
|
||||
&args.workdir,
|
||||
))
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn validate_workspace(_args: &[String]) -> Result<()> {
|
||||
Err(miette::miette!(
|
||||
"workspace validation is only supported on Unix"
|
||||
))
|
||||
}
|
||||
|
||||
/// Copy the running executable to `dest`, creating parent directories as
|
||||
/// needed and ensuring the result is executable (mode `0755`).
|
||||
///
|
||||
/// If `dest` already exists as a directory, the binary is placed inside it
|
||||
/// using the source executable's file name. This mirrors `cp` semantics so
|
||||
/// callers can pass either a full target path or a directory.
|
||||
fn copy_self(dest: &str) -> Result<()> {
|
||||
let exe = std::env::current_exe().into_diagnostic()?;
|
||||
|
||||
let dest_path = Path::new(dest);
|
||||
let final_path = if dest_path.is_dir() {
|
||||
let file_name = exe
|
||||
.file_name()
|
||||
.ok_or_else(|| miette::miette!("current_exe has no file name: {}", exe.display()))?;
|
||||
dest_path.join(file_name)
|
||||
} else {
|
||||
dest_path.to_path_buf()
|
||||
};
|
||||
|
||||
if let Some(parent) = final_path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
std::fs::create_dir_all(parent).into_diagnostic()?;
|
||||
}
|
||||
|
||||
std::fs::copy(&exe, &final_path).into_diagnostic()?;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mut perms = std::fs::metadata(&final_path)
|
||||
.into_diagnostic()?
|
||||
.permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&final_path, perms).into_diagnostic()?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> {
|
||||
use miette::Context as _;
|
||||
use nix::unistd::{Gid, Uid, chown};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
std::fs::create_dir_all(path)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?;
|
||||
let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions();
|
||||
perms.set_mode(mode);
|
||||
std::fs::set_permissions(path, perms)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?;
|
||||
chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid)))
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to chown sidecar directory {} to {uid}:{gid}",
|
||||
path.display()
|
||||
)
|
||||
})?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> {
|
||||
use miette::Context as _;
|
||||
use nix::unistd::{Gid, Uid, chown};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let uid = Uid::current();
|
||||
let gid = Gid::current();
|
||||
std::fs::create_dir_all(path)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?;
|
||||
chown(path, Some(uid), Some(gid))
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to chown sidecar directory {} to {}:{}",
|
||||
path.display(),
|
||||
uid.as_raw(),
|
||||
gid.as_raw()
|
||||
)
|
||||
})?;
|
||||
let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions();
|
||||
perms.set_mode(mode);
|
||||
std::fs::set_permissions(path, perms)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn copy_sidecar_client_tls_if_present(
|
||||
source_dir: &Path,
|
||||
sidecar_tls_dir: &Path,
|
||||
uid: u32,
|
||||
gid: u32,
|
||||
) -> Result<()> {
|
||||
use miette::Context as _;
|
||||
use nix::unistd::{Gid, Uid, chown};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
if !source_dir.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR);
|
||||
prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?;
|
||||
for file_name in CLIENT_TLS_FILES {
|
||||
let source = source_dir.join(file_name);
|
||||
if !source.exists() {
|
||||
return Err(miette::miette!(
|
||||
"client TLS source file is missing: {}",
|
||||
source.display()
|
||||
));
|
||||
}
|
||||
let dest = dest_dir.join(file_name);
|
||||
if dest.exists() {
|
||||
std::fs::remove_file(&dest)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!("failed to remove stale client TLS file {}", dest.display())
|
||||
})?;
|
||||
}
|
||||
std::fs::copy(&source, &dest)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to copy client TLS file {} to {}",
|
||||
source.display(),
|
||||
dest.display()
|
||||
)
|
||||
})?;
|
||||
let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions();
|
||||
perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE);
|
||||
std::fs::set_permissions(&dest, perms)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!("failed to chmod copied client TLS file {}", dest.display())
|
||||
})?;
|
||||
chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid)))
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to chown copied client TLS file {} to {uid}:{gid}",
|
||||
dest.display()
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn run_network_init(
|
||||
proxy_user_id: u32,
|
||||
proxy_primary_group_id: u32,
|
||||
sidecar_state_dir: &str,
|
||||
sidecar_tls_dir: &str,
|
||||
) -> Result<()> {
|
||||
validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?;
|
||||
|
||||
let sidecar_state_dir = Path::new(sidecar_state_dir);
|
||||
let sidecar_tls_dir = Path::new(sidecar_tls_dir);
|
||||
prepare_sidecar_directory(
|
||||
sidecar_state_dir,
|
||||
proxy_user_id,
|
||||
proxy_primary_group_id,
|
||||
SIDECAR_STATE_DIR_MODE,
|
||||
)?;
|
||||
// The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the
|
||||
// TLS work directory owned by the init user until the client cert copy is
|
||||
// complete, then hand it to the long-running proxy UID.
|
||||
prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?;
|
||||
copy_sidecar_client_tls_if_present(
|
||||
Path::new(CLIENT_TLS_DIR),
|
||||
sidecar_tls_dir,
|
||||
proxy_user_id,
|
||||
proxy_primary_group_id,
|
||||
)?;
|
||||
prepare_sidecar_directory(
|
||||
sidecar_tls_dir,
|
||||
proxy_user_id,
|
||||
proxy_primary_group_id,
|
||||
SIDECAR_TLS_DIR_MODE,
|
||||
)?;
|
||||
openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> {
|
||||
if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID {
|
||||
return Err(miette::miette!(
|
||||
"--proxy-uid must be 0 or at least {}",
|
||||
openshell_policy::MIN_SANDBOX_UID
|
||||
));
|
||||
}
|
||||
if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID {
|
||||
return Err(miette::miette!(
|
||||
"--proxy-gid must be at least {}",
|
||||
openshell_policy::MIN_SANDBOX_UID
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
fn run_network_init(
|
||||
_proxy_uid: u32,
|
||||
_proxy_gid: u32,
|
||||
_sidecar_state_dir: &str,
|
||||
_sidecar_tls_dir: &str,
|
||||
) -> Result<()> {
|
||||
Err(miette::miette!(
|
||||
"--mode=network-init is only supported on Linux"
|
||||
))
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// Handle `copy-self <DEST>` before clap so it works without any of the
|
||||
// sandbox flags. Kubernetes init containers invoke this path to seed an
|
||||
// emptyDir volume that the agent container then executes from.
|
||||
let raw_args: Vec<String> = std::env::args().collect();
|
||||
if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) {
|
||||
let dest = raw_args.get(2).ok_or_else(|| {
|
||||
miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} <DEST>")
|
||||
})?;
|
||||
return copy_self(dest);
|
||||
}
|
||||
|
||||
// Handle `debug-rpc <subcommand> [args]` before clap. Uses a small
|
||||
// dedicated runtime so we don't pay the supervisor's full startup cost.
|
||||
if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.into_diagnostic()?;
|
||||
return runtime.block_on(async move {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?;
|
||||
std::process::exit(exit);
|
||||
});
|
||||
}
|
||||
if raw_args.get(1).map(String::as_str) == Some(VALIDATE_WORKSPACE_SUBCOMMAND) {
|
||||
return validate_workspace(&raw_args[2..]);
|
||||
}
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
if args.mode.network_init {
|
||||
let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid);
|
||||
return run_network_init(
|
||||
args.proxy_uid,
|
||||
proxy_gid,
|
||||
&args.sidecar_state_dir,
|
||||
&args.sidecar_tls_dir,
|
||||
);
|
||||
}
|
||||
|
||||
// Try to open a rolling log file; fall back to stderr-only logging if it fails
|
||||
// (e.g., /var/log is not writable in custom workload images).
|
||||
// Rotates daily, keeps the 3 most recent files to bound disk usage.
|
||||
let file_logging = tracing_appender::rolling::RollingFileAppender::builder()
|
||||
.rotation(tracing_appender::rolling::Rotation::DAILY)
|
||||
.filename_prefix("openshell")
|
||||
.filename_suffix("log")
|
||||
.max_log_files(3)
|
||||
.build("/var/log")
|
||||
.ok()
|
||||
.map(|roller| {
|
||||
let (writer, guard) = tracing_appender::non_blocking(roller);
|
||||
(writer, guard)
|
||||
});
|
||||
|
||||
let console_filter =
|
||||
EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level));
|
||||
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.into_diagnostic()?;
|
||||
|
||||
let exit_code = runtime.block_on(async move {
|
||||
// Install rustls crypto provider before any TLS connections (including log push).
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
|
||||
// Set up optional log push layer (gRPC mode only).
|
||||
let log_push_state = if let (Some(sandbox_id), Some(endpoint)) =
|
||||
(&args.sandbox_id, &args.openshell_endpoint)
|
||||
{
|
||||
let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task(
|
||||
endpoint.clone(),
|
||||
sandbox_id.clone(),
|
||||
);
|
||||
let layer =
|
||||
openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx);
|
||||
Some((layer, handle))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone());
|
||||
let _log_push_handle = log_push_state.map(|(_, handle)| handle);
|
||||
|
||||
// Shared flag: the sandbox poll loop toggles this when the
|
||||
// `ocsf_json_enabled` setting changes. The JSONL layer checks it
|
||||
// on each event and short-circuits when false.
|
||||
let ocsf_enabled = Arc::new(AtomicBool::new(false));
|
||||
|
||||
// Keep guards alive for the entire process. When a guard is dropped the
|
||||
// non-blocking writer flushes remaining logs.
|
||||
let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging {
|
||||
let file_filter = EnvFilter::new("info");
|
||||
|
||||
// OCSF JSONL file: rolling appender matching the main log file
|
||||
// (daily rotation, 3 files max). Created eagerly but gated by the
|
||||
// enabled flag — no JSONL is written until ocsf_json_enabled is set.
|
||||
let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder()
|
||||
.rotation(tracing_appender::rolling::Rotation::DAILY)
|
||||
.filename_prefix("openshell-ocsf")
|
||||
.filename_suffix("log")
|
||||
.max_log_files(3)
|
||||
.build("/var/log")
|
||||
.ok()
|
||||
.map(|roller| {
|
||||
let (writer, guard) = tracing_appender::non_blocking(roller);
|
||||
let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone());
|
||||
(layer, guard)
|
||||
});
|
||||
let (jsonl_layer, jsonl_guard) = match jsonl_logging {
|
||||
Some((layer, guard)) => (Some(layer), Some(guard)),
|
||||
None => (None, None),
|
||||
};
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(
|
||||
OcsfShorthandLayer::new(std::io::stderr())
|
||||
.with_non_ocsf(true)
|
||||
.with_filter(console_filter),
|
||||
)
|
||||
.with(
|
||||
OcsfShorthandLayer::new(file_writer)
|
||||
.with_non_ocsf(true)
|
||||
.with_filter(file_filter),
|
||||
)
|
||||
.with(jsonl_layer.with_filter(LevelFilter::INFO))
|
||||
.with(push_layer.clone())
|
||||
.init();
|
||||
(Some(file_guard), jsonl_guard)
|
||||
} else {
|
||||
tracing_subscriber::registry()
|
||||
.with(
|
||||
OcsfShorthandLayer::new(std::io::stderr())
|
||||
.with_non_ocsf(true)
|
||||
.with_filter(console_filter),
|
||||
)
|
||||
.with(push_layer)
|
||||
.init();
|
||||
// Log the warning after the subscriber is initialized
|
||||
warn!("Could not open /var/log for log rotation; using stderr-only logging");
|
||||
(None, None)
|
||||
};
|
||||
|
||||
// Get command - either from CLI args, environment variable, or default to /bin/bash
|
||||
let command = if !args.command.is_empty() {
|
||||
args.command
|
||||
} else if let Ok(c) = std::env::var(openshell_core::sandbox_env::SANDBOX_COMMAND) {
|
||||
// Simple shell-like splitting on whitespace
|
||||
c.split_whitespace().map(String::from).collect()
|
||||
} else {
|
||||
vec!["/bin/bash".to_string()]
|
||||
};
|
||||
|
||||
info!(command = ?command, "Starting sandbox");
|
||||
// Note: "Starting sandbox" stays as plain info!() since the OCSF context
|
||||
// is not yet initialized at this point (run_sandbox hasn't been called).
|
||||
// The shorthand layer will render it in fallback format.
|
||||
|
||||
let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs {
|
||||
https_proxy: args.upstream_proxy,
|
||||
no_proxy: args.upstream_no_proxy,
|
||||
proxy_auth_file: args.upstream_proxy_auth_file,
|
||||
proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure,
|
||||
proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname,
|
||||
};
|
||||
|
||||
run_sandbox(
|
||||
command,
|
||||
args.workdir,
|
||||
args.timeout,
|
||||
args.interactive,
|
||||
args.sandbox_id,
|
||||
args.sandbox,
|
||||
args.openshell_endpoint,
|
||||
args.policy_rules,
|
||||
args.policy_data,
|
||||
args.ssh_socket_path,
|
||||
args.health_check,
|
||||
args.health_port,
|
||||
args.inference_routes,
|
||||
ocsf_enabled,
|
||||
args.mode.network,
|
||||
args.mode.process,
|
||||
upstream_proxy_args,
|
||||
)
|
||||
.await
|
||||
})?;
|
||||
|
||||
std::process::exit(exit_code);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn workspace_validation_subcommand_uses_final_policy_identity() {
|
||||
let uid = nix::unistd::geteuid().as_raw();
|
||||
let gid = nix::unistd::getegid().as_raw();
|
||||
if uid < 1000 || gid < 1000 {
|
||||
return;
|
||||
}
|
||||
let dir = tempfile::tempdir_in("/tmp").unwrap();
|
||||
std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap();
|
||||
let root = dir.path().canonicalize().unwrap().join("workspace");
|
||||
std::fs::create_dir(&root).unwrap();
|
||||
std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
let args = vec![
|
||||
"--workdir".to_string(),
|
||||
root.display().to_string(),
|
||||
"--expected-uid".to_string(),
|
||||
uid.to_string(),
|
||||
"--expected-gid".to_string(),
|
||||
gid.to_string(),
|
||||
];
|
||||
|
||||
validate_workspace(&args).expect("current identity should retain workspace authority");
|
||||
}
|
||||
|
||||
/// Drives `copy_self`'s file-copy logic against an arbitrary source path
|
||||
/// so tests don't depend on `current_exe()`.
|
||||
fn copy_executable(src: &Path, dest: &Path) -> Result<()> {
|
||||
let final_path = if dest.is_dir() {
|
||||
dest.join(src.file_name().unwrap())
|
||||
} else {
|
||||
dest.to_path_buf()
|
||||
};
|
||||
if let Some(parent) = final_path.parent()
|
||||
&& !parent.as_os_str().is_empty()
|
||||
{
|
||||
std::fs::create_dir_all(parent).into_diagnostic()?;
|
||||
}
|
||||
std::fs::copy(src, &final_path).into_diagnostic()?;
|
||||
let mut perms = std::fs::metadata(&final_path)
|
||||
.into_diagnostic()?
|
||||
.permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&final_path, perms).into_diagnostic()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_self_writes_executable_at_target_path() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let src = tmp.path().join("source-bin");
|
||||
std::fs::write(&src, b"#!/bin/false\n").unwrap();
|
||||
|
||||
let dest = tmp.path().join("subdir/openshell-sandbox");
|
||||
copy_executable(&src, &dest).unwrap();
|
||||
|
||||
assert!(dest.exists(), "destination file should exist");
|
||||
let mode = std::fs::metadata(&dest).unwrap().permissions().mode();
|
||||
assert_eq!(mode & 0o777, 0o755, "destination must be 0755");
|
||||
let copied = std::fs::read(&dest).unwrap();
|
||||
assert_eq!(copied, b"#!/bin/false\n");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn copy_self_into_existing_directory_uses_source_filename() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let src = tmp.path().join("openshell-sandbox");
|
||||
std::fs::write(&src, b"binary").unwrap();
|
||||
|
||||
let dest_dir = tmp.path().join("bin");
|
||||
std::fs::create_dir_all(&dest_dir).unwrap();
|
||||
|
||||
copy_executable(&src, &dest_dir).unwrap();
|
||||
|
||||
let final_path = dest_dir.join("openshell-sandbox");
|
||||
assert!(final_path.exists(), "binary should land inside dest dir");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_parses_network_init_standalone() {
|
||||
let mode = "network-init".parse::<Mode>().unwrap();
|
||||
assert!(mode.network_init);
|
||||
assert!(!mode.network);
|
||||
assert!(!mode.process);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_rejects_combined_network_init() {
|
||||
let err = "network-init,network".parse::<Mode>().unwrap_err();
|
||||
assert!(err.contains("cannot be combined"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mode_rejects_empty_value() {
|
||||
let err = "".parse::<Mode>().unwrap_err();
|
||||
assert!(err.contains("at least one"));
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() {
|
||||
assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755);
|
||||
assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700);
|
||||
assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750);
|
||||
assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400);
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() {
|
||||
validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap();
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn network_init_still_rejects_low_non_root_proxy_ids() {
|
||||
let uid_err =
|
||||
validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err();
|
||||
assert!(uid_err.to_string().contains("--proxy-uid"));
|
||||
|
||||
let gid_err = validate_network_init_ids(0, 999).unwrap_err();
|
||||
assert!(gid_err.to_string().contains("--proxy-gid"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
fn main() {
|
||||
eprintln!("openshell-sandbox supervisor is not available on Windows");
|
||||
std::process::exit(1);
|
||||
}
|
||||
Reference in New Issue
Block a user