trending/OpenShell
1
0
Fork 0
mirror of https://github.com/NVIDIA/OpenShell.git synced 2026-10-26 06:52:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(gpu): prefer single CDI devices for local runtimes (#1675)

Browse Source 
Prefer a single CDI-qualified device when Docker or Podman resolves the default GPU request to one GPU.

Allow nvidia.com/gpu=all only as a WSL2 all-only compatibility fallback, using Docker daemon info and Podman's /dev/dxg probe to identify that case.

Update driver docs, architecture notes, and GPU e2e coverage for the default selection behavior.

Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
Evan Lezar
2026-06-16 10:14:49 +02:00
committed by GitHub
parent f4a50059e3
commit 294c64eed1
10 changed files with 1156 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
architecture/compute-runtimes.md
+6
View File
@@ -55,6 +55,12 @@ through the driver configuration. The Helm chart defaults sandbox agents to
`Unconfined` so runtime/default AppArmor profiles do not block supervisor
network namespace setup on AppArmor-enabled nodes.
GPU requests enter the driver layer through `SandboxSpec.gpu` and
`SandboxSpec.gpu_device`. Docker and Podman map default GPU requests to one
concrete NVIDIA CDI device when individual CDI devices are available, use
`nvidia.com/gpu=all` only for WSL2/all-only compatibility, and pass explicit
driver-native device IDs through.
VM runtime state paths are derived only from driver-validated sandbox IDs
matching `[A-Za-z0-9._-]{1,128}`. The gateway-owned VM driver socket uses a
private `run/` directory plus Unix peer UID/PID checks. Standalone
crates/openshell-core/src/gpu.rs
+311 -12
View File
@@ -3,15 +3,184 @@
//! Shared GPU request helpers.
use std::fmt;
use std::sync::atomic::{AtomicUsize, Ordering};
use crate::config::CDI_GPU_DEVICE_ALL;
/// Resolve a GPU request into CDI device identifiers.
const CDI_NVIDIA_GPU_PREFIX: &str = "nvidia.com/gpu=";
const CDI_NVIDIA_GPU_ALL_SUFFIX: &str = "all";
/// Normalized CDI GPU inventory used by local container drivers.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct CdiGpuInventory {
device_ids: Vec<String>,
}
impl CdiGpuInventory {
/// Build a normalized inventory from runtime-reported CDI device IDs.
#[must_use]
pub fn new(device_ids: impl IntoIterator<Item = impl AsRef<str>>) -> Self {
let mut device_ids = device_ids
.into_iter()
.filter_map(|id| {
let id = id.as_ref().trim();
id.starts_with(CDI_NVIDIA_GPU_PREFIX)
.then(|| id.to_string())
})
.collect::<Vec<_>>();
device_ids.sort();
device_ids.dedup();
Self { device_ids }
}
#[must_use]
pub fn as_slice(&self) -> &[String] {
&self.device_ids
}
#[must_use]
pub fn is_empty(&self) -> bool {
self.device_ids.is_empty()
}
fn default_device_family(
&self,
allow_all_devices: bool,
) -> Result<Vec<String>, CdiGpuSelectionError> {
let mut indexed = self
.device_ids
.iter()
.filter_map(|id| {
let suffix = cdi_nvidia_gpu_suffix(id)?;
let index = suffix.parse::<u64>().ok()?;
Some((index, id.clone()))
})
.collect::<Vec<_>>();
if !indexed.is_empty() {
indexed.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1)));
return Ok(indexed.into_iter().map(|(_, id)| id).collect());
}
let mut named = self
.device_ids
.iter()
.filter_map(|id| {
let suffix = cdi_nvidia_gpu_suffix(id)?;
(suffix != CDI_NVIDIA_GPU_ALL_SUFFIX).then(|| id.clone())
})
.collect::<Vec<_>>();
if !named.is_empty() {
named.sort();
return Ok(named);
}
if self.device_ids.iter().any(|id| id == CDI_GPU_DEVICE_ALL) {
if !allow_all_devices {
return Err(CdiGpuSelectionError::AllDevicesDefaultUnsupported);
}
return Ok(vec![CDI_GPU_DEVICE_ALL.to_string()]);
}
Err(CdiGpuSelectionError::NoAvailableDevices)
}
}
/// Concurrency-safe round-robin cursor for default CDI GPU selection.
#[derive(Debug, Default)]
pub struct CdiGpuRoundRobin {
next: AtomicUsize,
}
impl CdiGpuRoundRobin {
#[must_use]
pub const fn new() -> Self {
Self {
next: AtomicUsize::new(0),
}
}
/// Return the next default device ID and advance the cursor.
pub fn next_default_device_id(
&self,
inventory: &CdiGpuInventory,
allow_all_devices: bool,
) -> Result<String, CdiGpuSelectionError> {
self.selected_default_device_id(inventory, true, allow_all_devices)
}
/// Return the current default device ID without advancing the cursor.
pub fn peek_default_device_id(
&self,
inventory: &CdiGpuInventory,
allow_all_devices: bool,
) -> Result<String, CdiGpuSelectionError> {
self.selected_default_device_id(inventory, false, allow_all_devices)
}
fn selected_default_device_id(
&self,
inventory: &CdiGpuInventory,
consume: bool,
allow_all_devices: bool,
) -> Result<String, CdiGpuSelectionError> {
let devices = inventory.default_device_family(allow_all_devices)?;
let base = if consume {
self.next.fetch_add(1, Ordering::Relaxed)
} else {
self.next.load(Ordering::Relaxed)
};
Ok(devices[base % devices.len()].clone())
}
}
/// CDI GPU selection failed.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CdiGpuSelectionError {
NoAvailableDevices,
MissingDefaultDevice,
AllDevicesDefaultUnsupported,
}
impl fmt::Display for CdiGpuSelectionError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NoAvailableDevices => f.write_str("no NVIDIA CDI GPU devices were discovered"),
Self::MissingDefaultDevice => {
f.write_str("GPU request requires a selected default CDI GPU device")
}
Self::AllDevicesDefaultUnsupported => f.write_str(
"default GPU request resolved only to nvidia.com/gpu=all, which is not allowed on this platform; set driver_config.cdi_devices to [\"nvidia.com/gpu=all\"] explicitly to request all GPUs",
),
}
}
}
impl std::error::Error for CdiGpuSelectionError {}
/// Resolve a local runtime GPU request into CDI device identifiers.
///
/// `None` means no GPU was requested. A GPU request with no explicit CDI
/// devices uses the CDI all-GPU request; otherwise the driver-configured CDI
/// devices pass through unchanged.
/// `None` means no GPU was requested. Explicit driver-configured CDI devices
/// pass through unchanged. A default GPU request uses the driver-selected
/// default CDI ID.
pub fn cdi_gpu_device_ids(
gpu: bool,
cdi_devices: &[String],
selected_default_device: Option<&str>,
) -> Result<Option<Vec<String>>, CdiGpuSelectionError> {
if !gpu {
return Ok(None);
}
if !cdi_devices.is_empty() {
return Ok(Some(cdi_devices.to_vec()));
}
let device = selected_default_device.ok_or(CdiGpuSelectionError::MissingDefaultDevice)?;
Ok(Some(vec![device.to_string()]))
}
/// Resolve a GPU request with the legacy all-GPU default.
#[must_use]
pub fn cdi_gpu_device_ids(gpu: bool, cdi_devices: &[String]) -> Option<Vec<String>> {
pub fn cdi_gpu_device_ids_or_all(gpu: bool, cdi_devices: &[String]) -> Option<Vec<String>> {
gpu.then(|| {
if cdi_devices.is_empty() {
vec![CDI_GPU_DEVICE_ALL.to_string()]
@@ -21,20 +190,32 @@ pub fn cdi_gpu_device_ids(gpu: bool, cdi_devices: &[String]) -> Option<Vec<Strin
})
}
fn cdi_nvidia_gpu_suffix(id: &str) -> Option<&str> {
id.strip_prefix(CDI_NVIDIA_GPU_PREFIX)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn cdi_gpu_device_ids_returns_none_when_absent() {
assert_eq!(cdi_gpu_device_ids(false, &[]), None);
assert_eq!(cdi_gpu_device_ids(false, &[], None), Ok(None));
}
#[test]
fn cdi_gpu_device_ids_defaults_empty_request_to_all_gpus() {
fn cdi_gpu_device_ids_uses_selected_default_device() {
assert_eq!(
cdi_gpu_device_ids(true, &[]),
Some(vec![CDI_GPU_DEVICE_ALL.to_string()])
cdi_gpu_device_ids(true, &[], Some("nvidia.com/gpu=0")),
Ok(Some(vec!["nvidia.com/gpu=0".to_string()]))
);
}
#[test]
fn cdi_gpu_device_ids_rejects_missing_default_device() {
assert_eq!(
cdi_gpu_device_ids(true, &[], None),
Err(CdiGpuSelectionError::MissingDefaultDevice)
);
}
@@ -46,12 +227,130 @@ mod tests {
&[
"nvidia.com/gpu=0".to_string(),
"nvidia.com/gpu=1".to_string()
]
],
None
),
Some(vec![
Ok(Some(vec![
"nvidia.com/gpu=0".to_string(),
"nvidia.com/gpu=1".to_string()
])
]))
);
}
#[test]
fn cdi_gpu_device_ids_or_all_uses_all_when_no_devices_are_configured() {
assert_eq!(
cdi_gpu_device_ids_or_all(true, &[]),
Some(vec![CDI_GPU_DEVICE_ALL.to_string()])
);
}
#[test]
fn inventory_filters_and_deduplicates_nvidia_gpu_ids() {
let inventory = CdiGpuInventory::new([
"nvidia.com/gpu=1",
"vendor.example/device=0",
"nvidia.com/gpu=1",
" nvidia.com/gpu=0 ",
]);
assert_eq!(
inventory.as_slice(),
&vec![
"nvidia.com/gpu=0".to_string(),
"nvidia.com/gpu=1".to_string()
]
);
}
#[test]
fn round_robin_prefers_indexed_family_and_sorts_numerically() {
let inventory = CdiGpuInventory::new([
"nvidia.com/gpu=10",
"nvidia.com/gpu=UUID-b",
"nvidia.com/gpu=2",
"nvidia.com/gpu=all",
]);
let selector = CdiGpuRoundRobin::new();
assert_eq!(
selector.next_default_device_id(&inventory, false),
Ok("nvidia.com/gpu=2".to_string())
);
assert_eq!(
selector.next_default_device_id(&inventory, false),
Ok("nvidia.com/gpu=10".to_string())
);
assert_eq!(
selector.next_default_device_id(&inventory, false),
Ok("nvidia.com/gpu=2".to_string())
);
}
#[test]
fn round_robin_uses_named_family_when_no_indexed_ids_exist() {
let inventory = CdiGpuInventory::new(["nvidia.com/gpu=UUID-b", "nvidia.com/gpu=UUID-a"]);
let selector = CdiGpuRoundRobin::new();
assert_eq!(
selector.next_default_device_id(&inventory, false),
Ok("nvidia.com/gpu=UUID-a".to_string())
);
}
#[test]
fn round_robin_uses_all_only_inventory_when_allowed() {
let inventory = CdiGpuInventory::new([CDI_GPU_DEVICE_ALL]);
let selector = CdiGpuRoundRobin::new();
assert_eq!(
selector.next_default_device_id(&inventory, true),
Ok(CDI_GPU_DEVICE_ALL.to_string())
);
}
#[test]
fn round_robin_rejects_all_only_inventory_when_not_allowed() {
let inventory = CdiGpuInventory::new([CDI_GPU_DEVICE_ALL]);
let selector = CdiGpuRoundRobin::new();
assert_eq!(
selector.next_default_device_id(&inventory, false),
Err(CdiGpuSelectionError::AllDevicesDefaultUnsupported)
);
}
#[test]
fn round_robin_rejects_empty_inventory() {
let inventory = CdiGpuInventory::new(["vendor.example/device=0"]);
let selector = CdiGpuRoundRobin::new();
assert_eq!(
selector.next_default_device_id(&inventory, false),
Err(CdiGpuSelectionError::NoAvailableDevices)
);
}
#[test]
fn peek_does_not_advance_round_robin_cursor() {
let inventory = CdiGpuInventory::new(["nvidia.com/gpu=0", "nvidia.com/gpu=1"]);
let selector = CdiGpuRoundRobin::new();
assert_eq!(
selector.peek_default_device_id(&inventory, false),
Ok("nvidia.com/gpu=0".to_string())
);
assert_eq!(
selector.peek_default_device_id(&inventory, false),
Ok("nvidia.com/gpu=0".to_string())
);
assert_eq!(
selector.next_default_device_id(&inventory, false),
Ok("nvidia.com/gpu=0".to_string())
);
assert_eq!(
selector.next_default_device_id(&inventory, false),
Ok("nvidia.com/gpu=1".to_string())
);
}
}
crates/openshell-driver-docker/README.md
+1 -1
View File
@@ -32,7 +32,7 @@ contract:
| `apparmor=unconfined` | Avoids Docker's default profile blocking required mount operations. |
| `restart_policy = unless-stopped` | Keeps managed sandboxes resumable across daemon or gateway restarts. |
| `PidsLimit` | Enforces the sandbox PID budget at the Docker cgroup layer. Set `[openshell.drivers.docker].sandbox_pids_limit = 0` to inherit the Docker/runtime default. |
| CDI GPU request | Uses `driver_config.cdi_devices` when set; otherwise requests all NVIDIA GPUs when the sandbox spec asks for GPU support and daemon CDI support is detected. |
| CDI GPU request | Uses `driver_config.cdi_devices` when set; otherwise selects one concrete NVIDIA CDI GPU when the sandbox spec asks for GPU support and daemon CDI support is detected. Docker daemon `/info` can permit `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback. |
The agent child process does not retain these supervisor privileges.
crates/openshell-driver-docker/src/lib.rs
+122 -14
View File
@@ -27,7 +27,9 @@ use openshell_core::driver_utils::{
LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME,
LABEL_SANDBOX_NAMESPACE, SUPERVISOR_IMAGE_BINARY_PATH, supervisor_image_should_refresh,
};
use openshell_core::gpu::cdi_gpu_device_ids;
use openshell_core::gpu::{
CdiGpuInventory, CdiGpuRoundRobin, CdiGpuSelectionError, cdi_gpu_device_ids,
};
use openshell_core::progress::{
PROGRESS_STEP_PULLING_IMAGE, PROGRESS_STEP_REQUESTING_SANDBOX, PROGRESS_STEP_STARTING_SANDBOX,
format_bytes, mark_progress_active, mark_progress_complete, mark_progress_detail,
@@ -231,6 +233,8 @@ struct DockerDriverRuntimeConfig {
guest_tls: Option<DockerGuestTlsPaths>,
daemon_version: String,
supports_gpu: bool,
cdi_gpu_inventory: CdiGpuInventory,
allow_all_default_gpu: bool,
sandbox_pids_limit: i64,
enable_bind_mounts: bool,
}
@@ -251,6 +255,7 @@ pub struct DockerComputeDriver {
events: broadcast::Sender<WatchSandboxesEvent>,
pending: Arc<Mutex<HashMap<String, PendingSandboxRecord>>>,
supervisor_readiness: Arc<dyn SupervisorReadiness>,
gpu_selector: Arc<CdiGpuRoundRobin>,
}
struct PendingSandboxRecord {
@@ -365,6 +370,8 @@ impl DockerComputeDriver {
.cdi_spec_dirs
.as_ref()
.is_some_and(|dirs| !dirs.is_empty());
let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info);
let allow_all_default_gpu = docker_info_reports_wsl2(&info);
validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?;
let gateway_port = config.bind_address.port();
if gateway_port == 0 {
@@ -412,12 +419,15 @@ impl DockerComputeDriver {
guest_tls,
daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()),
supports_gpu,
cdi_gpu_inventory,
allow_all_default_gpu,
sandbox_pids_limit: docker_config.sandbox_pids_limit,
enable_bind_mounts: docker_config.enable_bind_mounts,
},
events: broadcast::channel(WATCH_BUFFER).0,
pending: Arc::new(Mutex::new(HashMap::new())),
supervisor_readiness,
gpu_selector: Arc::new(CdiGpuRoundRobin::new()),
};
let poll_driver = driver.clone();
@@ -564,6 +574,43 @@ impl DockerComputeDriver {
Ok(())
}
fn peek_default_gpu_device(&self, sandbox: &DriverSandbox) -> Result<Option<String>, Status> {
self.selected_default_gpu_device(sandbox, false)
}
fn next_default_gpu_device(&self, sandbox: &DriverSandbox) -> Result<Option<String>, Status> {
self.selected_default_gpu_device(sandbox, true)
}
fn selected_default_gpu_device(
&self,
sandbox: &DriverSandbox,
consume: bool,
) -> Result<Option<String>, Status> {
let Some(spec) = sandbox.spec.as_ref() else {
return Ok(None);
};
let driver_config =
DockerSandboxDriverConfig::from_sandbox(sandbox).map_err(Status::invalid_argument)?;
if !spec.gpu || driver_config.cdi_devices.is_some() {
return Ok(None);
}
let selected = if consume {
self.gpu_selector.next_default_device_id(
&self.config.cdi_gpu_inventory,
self.config.allow_all_default_gpu,
)
} else {
self.gpu_selector.peek_default_device_id(
&self.config.cdi_gpu_inventory,
self.config.allow_all_default_gpu,
)
}
.map_err(docker_gpu_selection_status)?;
Ok(Some(selected))
}
async fn get_sandbox_snapshot(
&self,
sandbox_id: &str,
@@ -602,7 +649,12 @@ impl DockerComputeDriver {
Self::validate_sandbox(sandbox, &self.config)?;
Self::validate_sandbox_auth(sandbox)?;
self.validate_user_volume_mounts_available(sandbox).await?;
let _ = build_container_create_body(sandbox, &self.config)?;
let selected_default_gpu = self.peek_default_gpu_device(sandbox)?;
let _ = build_container_create_body_with_default(
sandbox,
&self.config,
selected_default_gpu.as_deref(),
)?;
if self
.find_managed_container_summary(&sandbox.id, &sandbox.name)
@@ -676,7 +728,18 @@ impl DockerComputeDriver {
})?;
let container_name = container_name_for_sandbox(sandbox);
let create_body = build_container_create_body(sandbox, &self.config).map_err(|status| {
let selected_default_gpu = self.next_default_gpu_device(sandbox).map_err(|status| {
if token_file_created {
cleanup_sandbox_token_file(sandbox, &self.config);
}
DockerProvisioningFailure::new("ContainerCreateFailed", status.message())
})?;
let create_body = build_container_create_body_with_default(
sandbox,
&self.config,
selected_default_gpu.as_deref(),
)
.map_err(|status| {
if token_file_created {
cleanup_sandbox_token_file(sandbox, &self.config);
}
@@ -1313,6 +1376,7 @@ impl ComputeDriver for DockerComputeDriver {
.ok_or_else(|| Status::invalid_argument("sandbox is required"))?;
Self::validate_sandbox(&sandbox, &self.config)?;
self.validate_user_volume_mounts_available(&sandbox).await?;
let _ = self.peek_default_gpu_device(&sandbox)?;
Ok(Response::new(ValidateSandboxCreateResponse {}))
}
@@ -2114,7 +2178,40 @@ fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig
.collect()
}
fn build_device_requests(sandbox: &DriverSandbox) -> Result<Option<Vec<DeviceRequest>>, Status> {
fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory {
CdiGpuInventory::new(
info.discovered_devices
.as_deref()
.unwrap_or_default()
.iter()
.filter(|device| device.source.as_deref() == Some("cdi"))
.filter_map(|device| device.id.as_deref()),
)
}
fn docker_info_reports_wsl2(info: &SystemInfo) -> bool {
[
info.kernel_version.as_deref(),
info.operating_system.as_deref(),
]
.into_iter()
.flatten()
.any(os_or_kernel_reports_wsl2)
}
fn os_or_kernel_reports_wsl2(value: &str) -> bool {
let value = value.to_ascii_lowercase();
value.contains("wsl2") || value.contains("microsoft-standard")
}
fn docker_gpu_selection_status(err: CdiGpuSelectionError) -> Status {
Status::failed_precondition(err.to_string())
}
fn build_device_requests(
sandbox: &DriverSandbox,
selected_default_device: Option<&str>,
) -> Result<Option<Vec<DeviceRequest>>, Status> {
let Some(spec) = sandbox.spec.as_ref() else {
return Ok(None);
};
@@ -2128,20 +2225,31 @@ fn build_device_requests(sandbox: &DriverSandbox) -> Result<Option<Vec<DeviceReq
));
}
Ok(
cdi_gpu_device_ids(spec.gpu, &cdi_devices).map(|device_ids| {
vec![DeviceRequest {
driver: Some("cdi".to_string()),
device_ids: Some(device_ids),
..Default::default()
}]
}),
)
cdi_gpu_device_ids(spec.gpu, &cdi_devices, selected_default_device)
.map(|device_ids| {
device_ids.map(|device_ids| {
vec![DeviceRequest {
driver: Some("cdi".to_string()),
device_ids: Some(device_ids),
..Default::default()
}]
})
})
.map_err(docker_gpu_selection_status)
}
#[cfg(test)]
fn build_container_create_body(
sandbox: &DriverSandbox,
config: &DockerDriverRuntimeConfig,
) -> Result<ContainerCreateBody, Status> {
build_container_create_body_with_default(sandbox, config, None)
}
fn build_container_create_body_with_default(
sandbox: &DriverSandbox,
config: &DockerDriverRuntimeConfig,
selected_default_device: Option<&str>,
) -> Result<ContainerCreateBody, Status> {
let spec = sandbox
.spec
@@ -2153,7 +2261,7 @@ fn build_container_create_body(
.ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?;
let resource_limits = docker_resource_limits(template)?;
let user_mounts = docker_driver_mounts(template, config.enable_bind_mounts)?;
let device_requests = build_device_requests(sandbox)?;
let device_requests = build_device_requests(sandbox, selected_default_device)?;
let mut labels = template.labels.clone();
labels.insert(
LABEL_MANAGED_BY.to_string(),
crates/openshell-driver-docker/src/tests.rs
+159 -5
View File
@@ -2,7 +2,7 @@
// SPDX-License-Identifier: Apache-2.0
use super::*;
use openshell_core::config::{CDI_GPU_DEVICE_ALL, DEFAULT_SERVER_PORT};
use openshell_core::config::DEFAULT_SERVER_PORT;
use openshell_core::driver_utils::{
LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME,
LABEL_SANDBOX_NAMESPACE,
@@ -17,7 +17,7 @@ use openshell_core::proto::compute::v1::{
};
use std::fs;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::sync::{LazyLock, Mutex};
use std::sync::{Arc, LazyLock, Mutex};
use tempfile::TempDir;
const TLS_MOUNT_DIR: &str = "/etc/openshell/tls/client";
@@ -104,6 +104,8 @@ fn runtime_config() -> DockerDriverRuntimeConfig {
}),
daemon_version: "28.0.0".to_string(),
supports_gpu: false,
cdi_gpu_inventory: CdiGpuInventory::default(),
allow_all_default_gpu: false,
sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT,
enable_bind_mounts: false,
}
@@ -161,6 +163,28 @@ fn inspected_volume(driver: &str, options: HashMap<String, String>) -> bollard::
}
}
struct DisconnectedSupervisorReadiness;
impl SupervisorReadiness for DisconnectedSupervisorReadiness {
fn is_supervisor_connected(&self, _sandbox_id: &str) -> bool {
false
}
}
fn test_driver_with_config(config: DockerDriverRuntimeConfig) -> DockerComputeDriver {
DockerComputeDriver {
docker: Arc::new(
Docker::connect_with_http("http://127.0.0.1:2375", 1, bollard::API_DEFAULT_VERSION)
.expect("construct test Docker client"),
),
config,
events: broadcast::channel(WATCH_BUFFER).0,
pending: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
supervisor_readiness: Arc::new(DisconnectedSupervisorReadiness),
gpu_selector: Arc::new(CdiGpuRoundRobin::new()),
}
}
#[test]
fn container_visible_endpoint_rewrites_loopback_hosts() {
assert_eq!(
@@ -1098,13 +1122,15 @@ fn validate_sandbox_auth_accepts_gateway_token() {
}
#[test]
fn build_container_create_body_maps_gpu_to_all_cdi_device() {
fn build_container_create_body_maps_default_gpu_to_selected_cdi_device() {
let mut config = runtime_config();
config.supports_gpu = true;
let mut sandbox = test_sandbox();
sandbox.spec.as_mut().unwrap().gpu = true;
let create_body = build_container_create_body(&sandbox, &config).unwrap();
let create_body =
build_container_create_body_with_default(&sandbox, &config, Some("nvidia.com/gpu=1"))
.unwrap();
let request = create_body
.host_config
.as_ref()
@@ -1115,7 +1141,24 @@ fn build_container_create_body_maps_gpu_to_all_cdi_device() {
assert_eq!(request.driver.as_deref(), Some("cdi"));
assert_eq!(
request.device_ids.as_ref().unwrap(),
&vec![CDI_GPU_DEVICE_ALL.to_string()]
&vec!["nvidia.com/gpu=1".to_string()]
);
}
#[test]
fn build_container_create_body_rejects_missing_default_cdi_device() {
let mut config = runtime_config();
config.supports_gpu = true;
let mut sandbox = test_sandbox();
sandbox.spec.as_mut().unwrap().gpu = true;
let err = build_container_create_body(&sandbox, &config).unwrap_err();
assert_eq!(err.code(), tonic::Code::FailedPrecondition);
assert!(
err.message().contains("selected default CDI GPU device"),
"unexpected error: {}",
err.message()
);
}
@@ -1172,6 +1215,117 @@ fn build_container_create_body_rejects_empty_cdi_devices() {
assert!(err.message().contains("non-empty list"));
}
#[test]
fn driver_default_gpu_selection_consumes_distinct_devices_for_creates() {
let mut config = runtime_config();
config.supports_gpu = true;
config.cdi_gpu_inventory = CdiGpuInventory::new(["nvidia.com/gpu=0", "nvidia.com/gpu=1"]);
let driver = test_driver_with_config(config);
let mut first_sandbox = test_sandbox();
first_sandbox.id = "sbx-first".to_string();
first_sandbox.name = "first".to_string();
first_sandbox.spec.as_mut().unwrap().gpu = true;
let mut second_sandbox = test_sandbox();
second_sandbox.id = "sbx-second".to_string();
second_sandbox.name = "second".to_string();
second_sandbox.spec.as_mut().unwrap().gpu = true;
DockerComputeDriver::validate_sandbox(&first_sandbox, &driver.config).unwrap();
assert_eq!(
driver.peek_default_gpu_device(&first_sandbox).unwrap(),
Some("nvidia.com/gpu=0".to_string())
);
let first_device = driver.next_default_gpu_device(&first_sandbox).unwrap();
let first_create_body = build_container_create_body_with_default(
&first_sandbox,
&driver.config,
first_device.as_deref(),
)
.unwrap();
DockerComputeDriver::validate_sandbox(&second_sandbox, &driver.config).unwrap();
assert_eq!(
driver.peek_default_gpu_device(&second_sandbox).unwrap(),
Some("nvidia.com/gpu=1".to_string())
);
let second_device = driver.next_default_gpu_device(&second_sandbox).unwrap();
let second_create_body = build_container_create_body_with_default(
&second_sandbox,
&driver.config,
second_device.as_deref(),
)
.unwrap();
let first_request = first_create_body
.host_config
.as_ref()
.and_then(|host_config| host_config.device_requests.as_ref())
.and_then(|requests| requests.first())
.expect("first default GPU request should add a Docker device request");
let second_request = second_create_body
.host_config
.as_ref()
.and_then(|host_config| host_config.device_requests.as_ref())
.and_then(|requests| requests.first())
.expect("second default GPU request should add a Docker device request");
assert_eq!(
first_request.device_ids.as_ref().unwrap(),
&vec!["nvidia.com/gpu=0".to_string()]
);
assert_eq!(
second_request.device_ids.as_ref().unwrap(),
&vec!["nvidia.com/gpu=1".to_string()]
);
}
#[test]
fn docker_info_reports_wsl2_from_kernel_version() {
let info = SystemInfo {
kernel_version: Some("5.15.153.1-microsoft-standard-WSL2".to_string()),
operating_system: Some("Docker Desktop".to_string()),
..Default::default()
};
assert!(docker_info_reports_wsl2(&info));
}
#[test]
fn docker_info_reports_wsl2_from_operating_system() {
let info = SystemInfo {
operating_system: Some("Ubuntu 24.04.4 LTS on WSL2".to_string()),
..Default::default()
};
assert!(docker_info_reports_wsl2(&info));
}
#[test]
fn docker_info_reports_wsl2_ignores_daemon_name_and_labels() {
let info = SystemInfo {
kernel_version: Some("6.8.0-60-generic".to_string()),
operating_system: Some("Ubuntu 24.04.4 LTS".to_string()),
name: Some("wsl-docker-daemon".to_string()),
labels: Some(vec!["com.example.platform=wsl2".to_string()]),
..Default::default()
};
assert!(!docker_info_reports_wsl2(&info));
}
#[test]
fn docker_info_reports_wsl2_rejects_plain_linux() {
let info = SystemInfo {
kernel_version: Some("6.8.0-60-generic".to_string()),
operating_system: Some("Ubuntu 24.04.4 LTS".to_string()),
os_type: Some("linux".to_string()),
architecture: Some("x86_64".to_string()),
..Default::default()
};
assert!(!docker_info_reports_wsl2(&info));
}
#[test]
fn require_sandbox_identifier_rejects_when_id_and_name_are_empty() {
// Regression test: `delete_sandbox` (and the other identifier-keyed
crates/openshell-driver-podman/README.md
+1 -1
View File
@@ -46,7 +46,7 @@ The container spec in `container.rs` sets these security-critical fields:
| `no_new_privileges` | `true` | Prevents privilege escalation after exec. |
| `seccomp_profile_path` | `unconfined` | The supervisor installs its own policy-aware BPF filter. A container-level profile can block Landlock/seccomp syscalls during setup. |
| `mounts` | Private tmpfs at `/run/netns` | Lets the supervisor create named network namespaces in rootless Podman. |
| CDI GPU devices | `driver_config.cdi_devices` when set, otherwise all NVIDIA GPUs | Exposes requested GPUs to GPU-enabled sandbox containers. |
| CDI GPU devices | `driver_config.cdi_devices` when set, otherwise one concrete NVIDIA CDI GPU. Local `/dev/dxg` permits `nvidia.com/gpu=all` as a WSL2 all-only compatibility fallback. | Exposes requested GPUs to GPU-enabled sandbox containers. |
The restricted agent child does not retain these supervisor privileges.
crates/openshell-driver-podman/src/container.rs
+52 -14
View File
@@ -477,7 +477,10 @@ fn podman_pids_limit(value: i64) -> Option<i64> {
}
/// Build CDI GPU device list if GPU is requested.
fn build_devices(sandbox: &DriverSandbox) -> Result<Option<Vec<LinuxDevice>>, ComputeDriverError> {
fn build_devices(
sandbox: &DriverSandbox,
selected_default_device: Option<&str>,
) -> Result<Option<Vec<LinuxDevice>>, ComputeDriverError> {
let Some(spec) = sandbox.spec.as_ref() else {
return Ok(None);
};
@@ -490,14 +493,16 @@ fn build_devices(sandbox: &DriverSandbox) -> Result<Option<Vec<LinuxDevice>>, Co
));
}
Ok(
cdi_gpu_device_ids(spec.gpu, &cdi_devices).map(|device_ids| {
device_ids
.into_iter()
.map(|path| LinuxDevice { path })
.collect()
}),
)
cdi_gpu_device_ids(spec.gpu, &cdi_devices, selected_default_device)
.map(|device_ids| {
device_ids.map(|device_ids| {
device_ids
.into_iter()
.map(|path| LinuxDevice { path })
.collect()
})
})
.map_err(|err| ComputeDriverError::Precondition(err.to_string()))
}
pub fn podman_driver_volume_mount_sources(
@@ -784,10 +789,20 @@ pub fn build_container_spec_with_token(
.expect("container spec should be valid")
}
#[cfg(test)]
pub fn try_build_container_spec_with_token(
sandbox: &DriverSandbox,
config: &PodmanComputeConfig,
token_host_path: Option<&Path>,
) -> Result<Value, ComputeDriverError> {
build_container_spec_with_token_and_gpu_default(sandbox, config, token_host_path, None)
}
pub fn build_container_spec_with_token_and_gpu_default(
sandbox: &DriverSandbox,
config: &PodmanComputeConfig,
token_host_path: Option<&Path>,
selected_default_device: Option<&str>,
) -> Result<Value, ComputeDriverError> {
let image = resolve_image(sandbox, config);
let name = container_name(&sandbox.name);
@@ -796,9 +811,9 @@ pub fn try_build_container_spec_with_token(
let env = build_env(sandbox, config, image);
let labels = build_labels(sandbox);
let resource_limits = build_resource_limits(sandbox, config);
let devices = build_devices(sandbox)?;
let user_mounts = podman_user_mounts(sandbox, config.enable_bind_mounts)
.map_err(ComputeDriverError::InvalidArgument)?;
let devices = build_devices(sandbox, selected_default_device)?;
// Network configuration -- always bridge mode.
// Matches libpod's network spec format `{name: {opts}}`; the unit-struct
@@ -1240,8 +1255,7 @@ mod tests {
}
#[test]
fn container_spec_maps_empty_gpu_request_to_all_cdi_device() {
use openshell_core::config::CDI_GPU_DEVICE_ALL;
fn container_spec_maps_empty_gpu_request_to_selected_default_cdi_device() {
use openshell_core::proto::compute::v1::DriverSandboxSpec;
let mut sandbox = test_sandbox("test-id", "test-name");
@@ -1250,14 +1264,38 @@ mod tests {
..Default::default()
});
let config = test_config();
let spec = build_container_spec(&sandbox, &config);
let spec = build_container_spec_with_token_and_gpu_default(
&sandbox,
&config,
None,
Some("nvidia.com/gpu=1"),
)
.unwrap();
assert_eq!(
spec["devices"][0]["path"].as_str(),
Some(CDI_GPU_DEVICE_ALL)
Some("nvidia.com/gpu=1")
);
}
#[test]
fn container_spec_rejects_missing_default_cdi_device() {
use openshell_core::proto::compute::v1::DriverSandboxSpec;
let mut sandbox = test_sandbox("test-id", "test-name");
sandbox.spec = Some(DriverSandboxSpec {
gpu: true,
..Default::default()
});
let config = test_config();
let err = build_container_spec_with_token_and_gpu_default(&sandbox, &config, None, None)
.unwrap_err();
assert!(matches!(err, ComputeDriverError::Precondition(_)));
assert!(err.to_string().contains("selected default CDI GPU device"));
}
#[test]
fn container_spec_passes_explicit_cdi_device_id_through() {
use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate};
crates/openshell-driver-podman/src/driver.rs
+349 -24
View File
@@ -10,9 +10,12 @@ use crate::watcher::{
self, WatchStream, driver_sandbox_from_inspect, driver_sandbox_from_list_entry,
};
use openshell_core::ComputeDriverError;
use openshell_core::config::CDI_GPU_DEVICE_ALL;
use openshell_core::driver_utils::supervisor_image_should_refresh;
use openshell_core::gpu::{CdiGpuInventory, CdiGpuRoundRobin, CdiGpuSelectionError};
use openshell_core::proto::compute::v1::{DriverSandbox, GetCapabilitiesResponse};
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;
use tracing::{info, warn};
@@ -34,6 +37,9 @@ pub struct PodmanComputeDriver {
/// The host's IP on the bridge network. Sandbox containers use this to
/// reach the gateway server when no explicit gRPC endpoint is configured.
network_gateway_ip: Option<String>,
gpu_inventory: CdiGpuInventory,
allow_all_default_gpu: bool,
gpu_selector: Arc<CdiGpuRoundRobin>,
}
impl std::fmt::Debug for PodmanComputeDriver {
@@ -42,6 +48,8 @@ impl std::fmt::Debug for PodmanComputeDriver {
.field("socket_path", &self.config.socket_path)
.field("default_image", &self.config.default_image)
.field("network_name", &self.config.network_name)
.field("gpu_inventory", &self.gpu_inventory)
.field("allow_all_default_gpu", &self.allow_all_default_gpu)
.finish()
}
}
@@ -126,6 +134,42 @@ fn cleanup_sandbox_token_file(sandbox_id: &str) {
}
}
fn local_podman_cdi_gpu_inventory_from(dev_root: &Path) -> CdiGpuInventory {
let mut device_ids = std::fs::read_dir(dev_root)
.ok()
.into_iter()
.flat_map(|entries| entries.filter_map(Result::ok))
.filter_map(|entry| {
let name = entry.file_name();
let name = name.to_str()?;
let index = name.strip_prefix("nvidia")?;
(!index.is_empty() && index.chars().all(|ch| ch.is_ascii_digit()))
.then(|| format!("nvidia.com/gpu={index}"))
})
.collect::<Vec<_>>();
if local_podman_all_gpu_default_supported_from(dev_root) {
device_ids.push(CDI_GPU_DEVICE_ALL.to_string());
}
CdiGpuInventory::new(device_ids)
}
fn local_podman_cdi_gpu_inventory() -> CdiGpuInventory {
local_podman_cdi_gpu_inventory_from(Path::new("/dev"))
}
fn local_podman_all_gpu_default_supported_from(dev_root: &Path) -> bool {
dev_root.join("dxg").exists()
}
fn local_podman_all_gpu_default_supported() -> bool {
local_podman_all_gpu_default_supported_from(Path::new("/dev"))
}
fn podman_gpu_selection_error(err: CdiGpuSelectionError) -> ComputeDriverError {
ComputeDriverError::Precondition(err.to_string())
}
impl PodmanComputeDriver {
/// Create a new driver, verifying the Podman socket is reachable.
pub async fn new(mut config: PodmanComputeConfig) -> Result<Self, PodmanApiError> {
@@ -223,6 +267,15 @@ impl PodmanComputeDriver {
"Bridge network ready"
);
let gpu_inventory = local_podman_cdi_gpu_inventory();
let allow_all_default_gpu = local_podman_all_gpu_default_supported();
if !gpu_inventory.is_empty() {
info!(
device_count = gpu_inventory.as_slice().len(),
"Discovered local Podman NVIDIA CDI GPU devices"
);
}
// Auto-detect the gRPC callback endpoint when not explicitly
// configured. Sandbox containers use host.containers.internal
// (injected via hostadd with host-gateway in the container spec)
@@ -251,6 +304,9 @@ impl PodmanComputeDriver {
client,
config,
network_gateway_ip,
gpu_inventory,
allow_all_default_gpu,
gpu_selector: Arc::new(CdiGpuRoundRobin::new()),
})
}
@@ -277,15 +333,6 @@ impl PodmanComputeDriver {
&self.config.default_image
}
/// Check whether GPU devices are available via CDI.
///
/// The Podman system info response doesn't directly list CDI devices in all
/// versions. As a heuristic, check if the NVIDIA device node exists (this
/// works for both rootful and rootless).
fn has_gpu_capacity() -> bool {
std::path::Path::new("/dev/nvidia0").exists()
}
/// Validate a sandbox before creation.
pub async fn validate_sandbox_create(
&self,
@@ -298,18 +345,47 @@ impl PodmanComputeDriver {
"driver_config.cdi_devices requires gpu=true".to_string(),
));
}
Self::validate_gpu_request(gpu_requested)?;
self.validate_user_volume_mounts_available(sandbox).await?;
let _ = self.peek_default_gpu_device(sandbox)?;
Ok(())
}
fn validate_gpu_request(gpu_requested: bool) -> Result<(), ComputeDriverError> {
if gpu_requested && !Self::has_gpu_capacity() {
return Err(ComputeDriverError::Precondition(
"GPU sandbox requested, but no NVIDIA GPU devices are available.".to_string(),
));
fn peek_default_gpu_device(
&self,
sandbox: &DriverSandbox,
) -> Result<Option<String>, ComputeDriverError> {
self.selected_default_gpu_device(sandbox, false)
}
fn next_default_gpu_device(
&self,
sandbox: &DriverSandbox,
) -> Result<Option<String>, ComputeDriverError> {
self.selected_default_gpu_device(sandbox, true)
}
fn selected_default_gpu_device(
&self,
sandbox: &DriverSandbox,
consume: bool,
) -> Result<Option<String>, ComputeDriverError> {
let Some(spec) = sandbox.spec.as_ref() else {
return Ok(None);
};
let driver_config = PodmanSandboxDriverConfig::from_sandbox(sandbox)?;
if !spec.gpu || driver_config.cdi_devices.is_some() {
return Ok(None);
}
Ok(())
let selected = if consume {
self.gpu_selector
.next_default_device_id(&self.gpu_inventory, self.allow_all_default_gpu)
} else {
self.gpu_selector
.peek_default_device_id(&self.gpu_inventory, self.allow_all_default_gpu)
}
.map_err(podman_gpu_selection_error)?;
Ok(Some(selected))
}
async fn validate_user_volume_mounts_available(
@@ -423,11 +499,27 @@ impl PodmanComputeDriver {
};
// 3. Create container.
let spec = container::try_build_container_spec_with_token(
let selected_default_gpu = match self.next_default_gpu_device(sandbox) {
Ok(device) => device,
Err(e) => {
let _ = self.client.remove_volume(&vol_name).await;
cleanup_sandbox_token_file(&sandbox.id);
return Err(e);
}
};
let spec = match container::build_container_spec_with_token_and_gpu_default(
sandbox,
&self.config,
token_host_path.as_deref(),
)?;
selected_default_gpu.as_deref(),
) {
Ok(spec) => spec,
Err(e) => {
let _ = self.client.remove_volume(&vol_name).await;
cleanup_sandbox_token_file(&sandbox.id);
return Err(e);
}
};
match self.client.create_container(&spec).await {
Ok(_) => {}
Err(PodmanApiError::Conflict(_)) => {
@@ -634,11 +726,29 @@ impl PodmanComputeDriver {
#[cfg(test)]
impl PodmanComputeDriver {
pub(crate) fn for_tests(config: PodmanComputeConfig) -> Self {
Self::for_tests_with_gpu_inventory(config, CdiGpuInventory::default())
}
pub(crate) fn for_tests_with_gpu_inventory(
config: PodmanComputeConfig,
gpu_inventory: CdiGpuInventory,
) -> Self {
Self::for_tests_with_gpu_inventory_and_all_fallback(config, gpu_inventory, false)
}
pub(crate) fn for_tests_with_gpu_inventory_and_all_fallback(
config: PodmanComputeConfig,
gpu_inventory: CdiGpuInventory,
allow_all_default_gpu: bool,
) -> Self {
let client = PodmanClient::new(config.socket_path.clone());
Self {
client,
config,
network_gateway_ip: None,
gpu_inventory,
allow_all_default_gpu,
gpu_selector: Arc::new(CdiGpuRoundRobin::new()),
}
}
}
@@ -697,8 +807,32 @@ mod tests {
use hyper::StatusCode;
use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate};
use std::collections::HashMap;
use std::fs;
use std::path::PathBuf;
fn cdi_devices_config(device_ids: &[&str]) -> prost_types::Struct {
prost_types::Struct {
fields: std::iter::once((
"cdi_devices".to_string(),
prost_types::Value {
kind: Some(prost_types::value::Kind::ListValue(
prost_types::ListValue {
values: device_ids
.iter()
.map(|device_id| prost_types::Value {
kind: Some(prost_types::value::Kind::StringValue(
(*device_id).to_string(),
)),
})
.collect(),
},
)),
},
))
.collect(),
}
}
#[test]
fn podman_driver_error_from_conflict() {
let err = ComputeDriverError::from(PodmanApiError::Conflict("exists".into()));
@@ -783,6 +917,197 @@ mod tests {
assert_eq!(cfg.grpc_endpoint, "https://gateway.internal:9000");
}
#[test]
fn local_podman_cdi_gpu_inventory_maps_nvidia_device_nodes() {
let root = std::env::temp_dir().join(format!(
"openshell-podman-gpu-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system time should be after unix epoch")
.as_nanos()
));
fs::create_dir(&root).expect("create temp dev root");
fs::write(root.join("nvidia2"), "").expect("create nvidia2");
fs::write(root.join("nvidiactl"), "").expect("create nvidiactl");
fs::write(root.join("nvidia0"), "").expect("create nvidia0");
let inventory = local_podman_cdi_gpu_inventory_from(&root);
fs::remove_dir_all(&root).expect("remove temp dev root");
assert_eq!(
inventory.as_slice(),
&vec![
"nvidia.com/gpu=0".to_string(),
"nvidia.com/gpu=2".to_string()
]
);
}
#[test]
fn local_podman_cdi_gpu_inventory_maps_dxg_to_all_gpu_fallback() {
let root = std::env::temp_dir().join(format!(
"openshell-podman-dxg-test-{}-{}",
std::process::id(),
std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.expect("system time should be after unix epoch")
.as_nanos()
));
fs::create_dir(&root).expect("create temp dev root");
fs::write(root.join("dxg"), "").expect("create dxg");
let inventory = local_podman_cdi_gpu_inventory_from(&root);
let allow_all_default = local_podman_all_gpu_default_supported_from(&root);
fs::remove_dir_all(&root).expect("remove temp dev root");
assert_eq!(inventory.as_slice(), &vec![CDI_GPU_DEVICE_ALL.to_string()]);
assert!(allow_all_default);
}
#[tokio::test]
async fn validate_sandbox_create_accepts_default_gpu_with_inventory() {
use openshell_core::proto::compute::v1::DriverSandboxSpec;
let driver = PodmanComputeDriver::for_tests_with_gpu_inventory(
PodmanComputeConfig::default(),
CdiGpuInventory::new(["nvidia.com/gpu=0"]),
);
let sandbox = DriverSandbox {
spec: Some(DriverSandboxSpec {
gpu: true,
..Default::default()
}),
..Default::default()
};
driver.validate_sandbox_create(&sandbox).await.unwrap();
}
#[tokio::test]
async fn validate_sandbox_create_accepts_all_only_inventory_when_dxg_fallback_allowed() {
use openshell_core::proto::compute::v1::DriverSandboxSpec;
let driver = PodmanComputeDriver::for_tests_with_gpu_inventory_and_all_fallback(
PodmanComputeConfig::default(),
CdiGpuInventory::new([CDI_GPU_DEVICE_ALL]),
true,
);
let sandbox = DriverSandbox {
spec: Some(DriverSandboxSpec {
gpu: true,
..Default::default()
}),
..Default::default()
};
driver.validate_sandbox_create(&sandbox).await.unwrap();
}
#[tokio::test]
async fn validate_sandbox_create_rejects_all_only_inventory_without_dxg_fallback() {
use openshell_core::proto::compute::v1::DriverSandboxSpec;
let driver = PodmanComputeDriver::for_tests_with_gpu_inventory(
PodmanComputeConfig::default(),
CdiGpuInventory::new([CDI_GPU_DEVICE_ALL]),
);
let sandbox = DriverSandbox {
spec: Some(DriverSandboxSpec {
gpu: true,
..Default::default()
}),
..Default::default()
};
let err = driver.validate_sandbox_create(&sandbox).await.unwrap_err();
assert!(err.to_string().contains("nvidia.com/gpu=all"));
}
#[tokio::test]
async fn validate_sandbox_create_passes_explicit_cdi_device_id_without_inventory() {
use openshell_core::proto::compute::v1::{DriverSandboxSpec, DriverSandboxTemplate};
let driver = PodmanComputeDriver::for_tests(PodmanComputeConfig::default());
let sandbox = DriverSandbox {
spec: Some(DriverSandboxSpec {
gpu: true,
template: Some(DriverSandboxTemplate {
driver_config: Some(cdi_devices_config(&["nvidia.com/gpu=0"])),
..Default::default()
}),
..Default::default()
}),
..Default::default()
};
driver.validate_sandbox_create(&sandbox).await.unwrap();
}
#[test]
fn driver_default_gpu_selection_consumes_distinct_devices_for_creates() {
use openshell_core::proto::compute::v1::DriverSandboxSpec;
let driver = PodmanComputeDriver::for_tests_with_gpu_inventory(
PodmanComputeConfig::default(),
CdiGpuInventory::new(["nvidia.com/gpu=0", "nvidia.com/gpu=1"]),
);
let first_sandbox = DriverSandbox {
id: "sbx-first".to_string(),
name: "first".to_string(),
spec: Some(DriverSandboxSpec {
gpu: true,
..Default::default()
}),
..Default::default()
};
let second_sandbox = DriverSandbox {
id: "sbx-second".to_string(),
name: "second".to_string(),
spec: Some(DriverSandboxSpec {
gpu: true,
..Default::default()
}),
..Default::default()
};
assert_eq!(
driver.peek_default_gpu_device(&first_sandbox).unwrap(),
Some("nvidia.com/gpu=0".to_string())
);
let first_device = driver.next_default_gpu_device(&first_sandbox).unwrap();
let first_spec = container::build_container_spec_with_token_and_gpu_default(
&first_sandbox,
&driver.config,
None,
first_device.as_deref(),
)
.unwrap();
assert_eq!(
driver.peek_default_gpu_device(&second_sandbox).unwrap(),
Some("nvidia.com/gpu=1".to_string())
);
let second_device = driver.next_default_gpu_device(&second_sandbox).unwrap();
let second_spec = container::build_container_spec_with_token_and_gpu_default(
&second_sandbox,
&driver.config,
None,
second_device.as_deref(),
)
.unwrap();
assert_eq!(
first_spec["devices"][0]["path"].as_str(),
Some("nvidia.com/gpu=0")
);
assert_eq!(
second_spec["devices"][0]["path"].as_str(),
Some("nvidia.com/gpu=1")
);
}
#[test]
fn supervisor_pull_policy_refreshes_mutable_tags_only() {
assert_eq!(
@@ -969,7 +1294,7 @@ mod tests {
api_path("/libpod/volumes/work-bind/json")
)]
);
let _ = std::fs::remove_file(socket_path);
let _ = fs::remove_file(socket_path);
}
#[tokio::test]
@@ -1006,7 +1331,7 @@ mod tests {
api_path("/libpod/volumes/work-rbind/json")
)]
);
let _ = std::fs::remove_file(socket_path);
let _ = fs::remove_file(socket_path);
}
#[tokio::test]
@@ -1032,7 +1357,7 @@ mod tests {
.expect("bind-backed volume should be allowed when bind mounts are enabled");
handle.await.expect("stub task should finish");
let _ = std::fs::remove_file(socket_path);
let _ = fs::remove_file(socket_path);
}
#[tokio::test]
@@ -1088,7 +1413,7 @@ mod tests {
),
]
);
let _ = std::fs::remove_file(socket_path);
let _ = fs::remove_file(socket_path);
}
#[tokio::test]
@@ -1140,6 +1465,6 @@ mod tests {
api_path(&format!("/libpod/volumes/{volume_name}"))
)]
);
let _ = std::fs::remove_file(socket_path);
let _ = fs::remove_file(socket_path);
}
}
docs/sandboxes/manage-sandboxes.mdx
+7 -3
View File
@@ -74,9 +74,13 @@ For Docker-backed sandboxes, GPU injection uses Docker CDI. If you enable Docker
CDI after the gateway starts, restart the gateway so OpenShell can detect the
updated Docker daemon capability.
Exact GPU device selection is driver-specific and requires `--gpu`. For Docker
or Podman, pass CDI IDs through `cdi_devices`. The top-level key must match the
active driver; replace `docker` with `podman` when using Podman:
Docker and Podman gateways map a default `--gpu` request to one concrete
NVIDIA CDI device when individual CDI devices are available. On WSL2 all-only
runtimes, the default can fall back to `nvidia.com/gpu=all`.
Exact GPU device selection is driver-specific and still requires `--gpu`. For
Docker or Podman, pass CDI IDs through `cdi_devices`. The top-level key must
match the active driver; replace `docker` with `podman` when using Podman:
```shell
openshell sandbox create \
e2e/rust/tests/gpu_device_selection.rs
+148 -13
View File
@@ -74,6 +74,35 @@ fn collect_cdi_gpu_device_ids_from_devices(devices: &[Value], device_ids: &mut V
}
}
fn local_podman_cdi_gpu_device_ids() -> Vec<String> {
let mut device_ids = std::fs::read_dir("/dev")
.ok()
.into_iter()
.flat_map(|entries| entries.filter_map(Result::ok))
.filter_map(|entry| {
let name = entry.file_name();
let name = name.to_str()?;
let index = name.strip_prefix("nvidia")?;
(!index.is_empty() && index.chars().all(|ch| ch.is_ascii_digit()))
.then(|| format!("{CDI_GPU_DEVICE_PREFIX}{index}"))
})
.collect::<Vec<_>>();
if local_podman_all_gpu_default_supported() {
device_ids.push(CDI_GPU_DEVICE_ALL.to_string());
}
device_ids.sort();
device_ids.dedup();
device_ids
}
fn local_podman_all_gpu_default_supported() -> bool {
std::path::Path::new("/dev/dxg").exists()
}
fn uses_local_podman_inventory(engine: &ContainerEngine) -> bool {
engine.name().rsplit('/').next() == Some("podman")
}
fn parse_cdi_gpu_device_ids(info: &Value) -> Vec<String> {
let mut device_ids = Vec::new();
@@ -89,8 +118,7 @@ fn parse_cdi_gpu_device_ids(info: &Value) -> Vec<String> {
device_ids
}
fn discovered_cdi_gpu_device_ids() -> Vec<String> {
let engine = ContainerEngine::from_env();
fn container_engine_info(engine: &ContainerEngine) -> Value {
let output = engine
.command()
.args(["info", "--format", "json"])
@@ -108,22 +136,103 @@ fn discovered_cdi_gpu_device_ids() -> Vec<String> {
combined
);
let info: Value = serde_json::from_slice(&output.stdout).unwrap_or_else(|err| {
serde_json::from_slice(&output.stdout).unwrap_or_else(|err| {
panic!(
"failed to parse {} info JSON: {err}\n{combined}",
engine.name()
)
});
})
}
fn info_string<'a>(info: &'a Value, keys: &[&str]) -> Option<&'a str> {
keys.iter()
.find_map(|key| info.get(*key).and_then(Value::as_str))
}
fn text_reports_wsl2(value: &str) -> bool {
let value = value.to_ascii_lowercase();
value.contains("wsl2") || value.contains("microsoft-standard")
}
fn docker_info_reports_wsl2(info: &Value) -> bool {
[
info_string(info, &["KernelVersion", "kernelVersion", "kernel_version"]),
info_string(
info,
&["OperatingSystem", "operatingSystem", "operating_system"],
),
]
.into_iter()
.flatten()
.any(text_reports_wsl2)
}
fn all_gpu_default_allowed() -> bool {
let engine = ContainerEngine::from_env();
if uses_local_podman_inventory(&engine) {
return local_podman_all_gpu_default_supported();
}
docker_info_reports_wsl2(&container_engine_info(&engine))
}
fn discovered_cdi_gpu_device_ids() -> Vec<String> {
let engine = ContainerEngine::from_env();
if uses_local_podman_inventory(&engine) {
let device_ids = local_podman_cdi_gpu_device_ids();
assert!(
!device_ids.is_empty(),
"local Podman GPU e2e tests require /dev/nvidiaN device nodes or \
/dev/dxg so bare --gpu can be mapped to a supported NVIDIA CDI device"
);
return device_ids;
}
let info = container_engine_info(&engine);
let device_ids = parse_cdi_gpu_device_ids(&info);
assert!(
!device_ids.is_empty(),
"{} info --format json did not report any discovered NVIDIA CDI GPU devices. \
Expected DiscoveredDevices entries with Source=cdi and ID like nvidia.com/gpu=all.",
Expected DiscoveredDevices entries with Source=cdi and ID like nvidia.com/gpu=0.",
engine.name()
);
device_ids
}
fn default_cdi_gpu_device_id(device_ids: &[String], allow_all_devices: bool) -> Option<String> {
let mut indexed = device_ids
.iter()
.filter_map(|device_id| {
let suffix = device_id.strip_prefix(CDI_GPU_DEVICE_PREFIX)?;
let index = suffix.parse::<u64>().ok()?;
Some((index, device_id.clone()))
})
.collect::<Vec<_>>();
if !indexed.is_empty() {
indexed.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1)));
return indexed.into_iter().map(|(_, device_id)| device_id).next();
}
let mut named = device_ids
.iter()
.filter(|device_id| {
device_id.starts_with(CDI_GPU_DEVICE_PREFIX)
&& device_id.as_str() != CDI_GPU_DEVICE_ALL
})
.cloned()
.collect::<Vec<_>>();
if !named.is_empty() {
named.sort();
return named.into_iter().next();
}
(allow_all_devices
&& device_ids
.iter()
.any(|device_id| device_id == CDI_GPU_DEVICE_ALL))
.then(|| CDI_GPU_DEVICE_ALL.to_string())
}
fn has_cdi_gpu_device(device_id: &str) -> bool {
discovered_cdi_gpu_device_ids()
.iter()
@@ -228,20 +337,20 @@ async fn sandbox_create_output(args: &[&str]) -> String {
}
#[tokio::test]
async fn gpu_request_without_device_matches_plain_all_gpu_container() {
if !has_cdi_gpu_device(CDI_GPU_DEVICE_ALL) {
eprintln!(
"skipping default GPU request test because {CDI_GPU_DEVICE_ALL} was not discovered"
);
async fn gpu_request_without_device_matches_plain_default_gpu_container() {
let device_ids = discovered_cdi_gpu_device_ids();
let Some(default_gpu_device) = default_cdi_gpu_device_id(&device_ids, all_gpu_default_allowed())
else {
eprintln!("skipping default GPU request test because no selectable GPU ID was discovered");
return;
}
};
let expected = runtime_gpu_lines(CDI_GPU_DEVICE_ALL);
let expected = runtime_gpu_lines(&default_gpu_device);
let actual = sandbox_gpu_lines(None).await;
assert_eq!(
actual, expected,
"default GPU request should expose the same GPU lines as a plain all-GPU container"
"default GPU request should expose the same GPU lines as a plain container using {default_gpu_device}"
);
}
@@ -380,3 +489,29 @@ fn parse_cdi_gpu_device_ids_ignores_unexpected_nested_devices() {
assert!(parse_cdi_gpu_device_ids(&info).is_empty());
}
#[test]
fn docker_info_reports_wsl2_uses_kernel_and_operating_system_only() {
let info = serde_json::json!({
"KernelVersion": "5.15.153.1-microsoft-standard-WSL2",
"OperatingSystem": "Docker Desktop",
"OSType": "linux",
"Name": "docker-daemon",
"Labels": ["com.example.platform=linux"]
});
assert!(docker_info_reports_wsl2(&info));
}
#[test]
fn docker_info_reports_wsl2_ignores_daemon_name_and_labels() {
let info = serde_json::json!({
"KernelVersion": "6.8.0-60-generic",
"OperatingSystem": "Ubuntu 24.04.4 LTS",
"OSType": "wsl",
"Name": "wsl-docker-daemon",
"Labels": ["com.example.platform=wsl2"]
});
assert!(!docker_info_reports_wsl2(&info));
}