diff --git a/crates/openshell-driver-docker/Cargo.toml b/crates/openshell-driver-docker/Cargo.toml index 296274bf8..9ee0a3a38 100644 --- a/crates/openshell-driver-docker/Cargo.toml +++ b/crates/openshell-driver-docker/Cargo.toml @@ -10,6 +10,9 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +path = "src/lib_entry.rs" + [dependencies] openshell-core = { path = "../openshell-core", default-features = false } serde = { workspace = true } diff --git a/crates/openshell-driver-docker/src/lib.rs b/crates/openshell-driver-docker/src/lib.rs index d57d949cd..dd4d9ef0f 100644 --- a/crates/openshell-driver-docker/src/lib.rs +++ b/crates/openshell-driver-docker/src/lib.rs @@ -1,11 +1,9 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -mod windows; +//! Docker compute driver. -#[cfg(target_os = "windows")] -pub use windows::{DockerComputeConfig, default_docker_supervisor_image}; +#![allow(clippy::result_large_err)] use bollard::Docker; use bollard::errors::Error as BollardError; diff --git a/crates/openshell-driver-docker/src/lib_entry.rs b/crates/openshell-driver-docker/src/lib_entry.rs new file mode 100644 index 000000000..4272e7c53 --- /dev/null +++ b/crates/openshell-driver-docker/src/lib_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("lib.rs"); + +#[cfg(target_os = "windows")] +include!("lib_win.rs"); diff --git a/crates/openshell-driver-docker/src/lib_unix.rs b/crates/openshell-driver-docker/src/lib_unix.rs deleted file mode 100644 index 2f89c2229..000000000 --- a/crates/openshell-driver-docker/src/lib_unix.rs +++ /dev/null @@ -1,3522 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Docker compute driver. - -#![allow(clippy::result_large_err)] - -use bollard::Docker; -use bollard::errors::Error as BollardError; -use bollard::models::{ - ContainerCreateBody, ContainerSummary, ContainerSummaryStateEnum, CreateImageInfo, - DeviceRequest, EndpointSettings, HostConfig, Mount, MountTmpfsOptions, MountTypeEnum, - MountVolumeOptions, NetworkCreateRequest, NetworkingConfig, ProgressDetail, RestartPolicy, - RestartPolicyNameEnum, SystemInfo, -}; -use bollard::query_parameters::{ - CreateContainerOptionsBuilder, CreateImageOptions, DownloadFromContainerOptionsBuilder, - ListContainersOptionsBuilder, RemoveContainerOptionsBuilder, StopContainerOptionsBuilder, -}; -use bytes::Bytes; -use futures::{Stream, StreamExt}; -use openshell_core::config::{ - DEFAULT_DOCKER_NETWORK_NAME, DEFAULT_SANDBOX_PIDS_LIMIT, DEFAULT_STOP_TIMEOUT_SECS, -}; -use openshell_core::driver_mounts; -use openshell_core::driver_utils::{ - LABEL_MANAGED_BY, LABEL_MANAGED_BY_VALUE, LABEL_SANDBOX_ID, LABEL_SANDBOX_NAME, - LABEL_SANDBOX_NAMESPACE, LABEL_SANDBOX_WORKSPACE, SUPERVISOR_IMAGE_BINARY_PATH, - supervisor_image_should_refresh, -}; -use openshell_core::gpu::{ - CdiGpuDefaultSelector, CdiGpuInventory, CdiGpuSelectionError, driver_gpu_requirements, - effective_driver_gpu_count, validate_specific_gpu_device_request, -}; -use openshell_core::progress::{ - PROGRESS_STEP_PULLING_IMAGE, PROGRESS_STEP_REQUESTING_SANDBOX, PROGRESS_STEP_STARTING_SANDBOX, - format_bytes, mark_progress_active, mark_progress_complete, mark_progress_detail, -}; -use openshell_core::proto::compute::v1::{ - CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, - DriverCondition, DriverPlatformEvent, DriverSandbox, DriverSandboxStatus, - DriverSandboxTemplate, GetCapabilitiesRequest, GetCapabilitiesResponse, GetSandboxRequest, - GetSandboxResponse, GpuResourceRequirements, ListSandboxesRequest, ListSandboxesResponse, - StopSandboxRequest, StopSandboxResponse, ValidateSandboxCreateRequest, - ValidateSandboxCreateResponse, WatchSandboxesDeletedEvent, WatchSandboxesEvent, - WatchSandboxesPlatformEvent, WatchSandboxesRequest, WatchSandboxesSandboxEvent, - compute_driver_server::ComputeDriver, watch_sandboxes_event, -}; -use openshell_core::proto_struct::{ - deserialize_optional_non_empty_string_list, struct_to_json_value, -}; -use openshell_core::{Config, Error, Result as CoreResult}; -use std::collections::{HashMap, HashSet}; -use std::io::Read; -use std::net::{IpAddr, SocketAddr}; -use std::path::{Path, PathBuf}; -use std::pin::Pin; -use std::sync::Arc; -use std::time::Duration; -use tokio::sync::{Mutex, broadcast, mpsc}; -use tokio::task::JoinHandle; -use tokio_stream::wrappers::ReceiverStream; -use tonic::{Request, Response, Status}; -use tracing::{info, warn}; -use url::Url; - -const WATCH_BUFFER: usize = 128; -const WATCH_POLL_INTERVAL: Duration = Duration::from_secs(2); -const WATCH_POLL_MAX_BACKOFF: Duration = Duration::from_secs(30); - -const SUPERVISOR_MOUNT_PATH: &str = openshell_core::driver_utils::SUPERVISOR_CONTAINER_BINARY; -const TLS_CA_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CA_MOUNT_PATH; -const TLS_CERT_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_CERT_MOUNT_PATH; -const TLS_KEY_MOUNT_PATH: &str = openshell_core::driver_utils::TLS_KEY_MOUNT_PATH; -const SANDBOX_TOKEN_MOUNT_PATH: &str = openshell_core::driver_utils::SANDBOX_TOKEN_MOUNT_PATH; -const SANDBOX_COMMAND: &str = "sleep infinity"; -const SUPERVISOR_PATH: &str = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; -const HOST_OPENSHELL_INTERNAL: &str = "host.openshell.internal"; -const HOST_DOCKER_INTERNAL: &str = "host.docker.internal"; -const DOCKER_NETWORK_DRIVER: &str = "bridge"; - -/// Queried by the Docker driver to decide when a sandbox's supervisor -/// relay is live. Implementations return `true` once a sandbox has an -/// active `ConnectSupervisor` session registered. -/// -/// The driver cannot observe the supervisor's SSH socket directly (it -/// lives inside the container), so it leans on this signal to flip the -/// Ready condition from `DependenciesNotReady` to `True`. -pub trait SupervisorReadiness: Send + Sync + 'static { - fn is_supervisor_connected(&self, sandbox_id: &str) -> bool; -} - -/// Gateway-local configuration for the Docker compute driver. -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct DockerComputeConfig { - /// Docker API Unix socket. When unset, use the socket selected by gateway - /// auto-detection, falling back to `/var/run/docker.sock` for an explicitly - /// configured Docker driver. - pub socket_path: Option, - - /// Default OCI image for sandboxes. - pub default_image: String, - - /// Image pull policy for sandbox images. - pub image_pull_policy: String, - - /// Namespace label applied to Docker sandboxes. - pub sandbox_namespace: String, - - /// Gateway gRPC endpoint the sandbox connects back to. - pub grpc_endpoint: String, - - /// Optional override for the Linux `openshell-sandbox` binary mounted into containers. - pub supervisor_bin: Option, - - /// Optional image used to extract the Linux `openshell-sandbox` binary. - /// Ignored when `supervisor_bin` is set. See `resolve_supervisor_bin` for - /// the full resolution order. - pub supervisor_image: Option, - - /// Host-side CA certificate for Docker sandbox mTLS. - pub guest_tls_ca: Option, - - /// Host-side client certificate for Docker sandbox mTLS. - pub guest_tls_cert: Option, - - /// Host-side private key for Docker sandbox mTLS. - pub guest_tls_key: Option, - - /// Docker bridge network that sandbox containers join. - pub network_name: String, - - /// Host gateway IP used for sandbox host aliases. - pub host_gateway_ip: String, - - /// Unix socket path the in-container supervisor bridges relay traffic to. - pub ssh_socket_path: String, - - /// Container cgroup PID limit for Docker-managed sandboxes. - /// - /// Set to `0` to leave Docker's runtime/default PID limit unchanged. - pub sandbox_pids_limit: i64, - - /// Allow sandbox requests to attach host bind mounts through - /// `template.driver_config`. - #[serde(default)] - pub enable_bind_mounts: bool, -} - -impl Default for DockerComputeConfig { - fn default() -> Self { - Self { - socket_path: None, - default_image: openshell_core::image::default_sandbox_image(), - image_pull_policy: String::new(), - sandbox_namespace: "default".to_string(), - grpc_endpoint: String::new(), - supervisor_bin: None, - supervisor_image: None, - guest_tls_ca: None, - guest_tls_cert: None, - guest_tls_key: None, - network_name: DEFAULT_DOCKER_NETWORK_NAME.to_string(), - host_gateway_ip: String::new(), - ssh_socket_path: "/run/openshell/ssh.sock".to_string(), - sandbox_pids_limit: DEFAULT_SANDBOX_PIDS_LIMIT, - enable_bind_mounts: false, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub(crate) struct DockerGuestTlsPaths { - pub(crate) ca: PathBuf, - pub(crate) cert: PathBuf, - pub(crate) key: PathBuf, -} - -#[derive(Debug, Clone)] -struct DockerDriverRuntimeConfig { - default_image: String, - image_pull_policy: String, - sandbox_namespace: String, - grpc_endpoint: String, - network_name: String, - gateway_route: DockerGatewayRoute, - ssh_socket_path: String, - stop_timeout_secs: u32, - log_level: String, - supervisor_bin: PathBuf, - guest_tls: Option, - daemon_version: String, - supports_gpu: bool, - allow_all_default_gpu: bool, - sandbox_pids_limit: i64, - enable_bind_mounts: bool, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum DockerGatewayRoute { - Bridge { - bind_address: SocketAddr, - host_alias_ip: IpAddr, - }, - HostGateway, -} - -#[derive(Clone)] -pub struct DockerComputeDriver { - docker: Arc, - config: DockerDriverRuntimeConfig, - events: broadcast::Sender, - pending: Arc>>, - supervisor_readiness: Arc, - gpu_selector: Arc, -} - -struct PendingSandboxRecord { - sandbox: DriverSandbox, - task: Option>, -} - -#[derive(Debug, Clone)] -struct DockerProvisioningFailure { - reason: &'static str, - message: String, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -struct DockerResourceLimits { - nano_cpus: Option, - memory_bytes: Option, -} - -#[derive(Debug, Clone, Default, serde::Deserialize)] -#[serde(default, deny_unknown_fields)] -struct DockerSandboxDriverConfig { - #[serde( - default, - deserialize_with = "deserialize_optional_non_empty_string_list" - )] - cdi_devices: Option>, - mounts: Vec, -} - -struct ValidatedDockerSandbox<'a> { - template: &'a DriverSandboxTemplate, - driver_config: DockerSandboxDriverConfig, - gpu_requirements: Option<&'a GpuResourceRequirements>, -} - -impl DockerSandboxDriverConfig { - fn from_template(template: &DriverSandboxTemplate) -> Result { - let Some(config) = template.driver_config.as_ref() else { - return Ok(Self::default()); - }; - - serde_json::from_value(struct_to_json_value(config)) - .map_err(|err| format!("invalid docker driver_config: {err}")) - } -} - -use openshell_core::driver_mounts::SelinuxLabel; - -#[derive(Debug, Clone, serde::Deserialize)] -#[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] -enum DockerDriverMountConfig { - Bind { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - selinux_label: Option, - }, - Volume { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - subpath: Option, - }, - Tmpfs { - target: String, - #[serde(default)] - options: Vec, - #[serde(default)] - size_bytes: Option, - #[serde(default)] - mode: Option, - }, - Image { - source: String, - target: String, - #[serde(default = "default_true")] - read_only: bool, - #[serde(default)] - subpath: Option, - }, -} - -fn default_true() -> bool { - true -} - -type WatchStream = - Pin> + Send + 'static>>; - -impl DockerComputeDriver { - pub async fn new( - config: &Config, - docker_config: &DockerComputeConfig, - supervisor_readiness: Arc, - ) -> CoreResult { - let socket_path = docker_config - .socket_path - .clone() - .or_else(openshell_core::config::detect_docker_socket) - .unwrap_or_else(|| PathBuf::from("/var/run/docker.sock")); - let socket_path_str = socket_path.to_str().ok_or_else(|| { - Error::config(format!( - "Docker socket path is not valid UTF-8: {}", - socket_path.display() - )) - })?; - let docker = - Docker::connect_with_socket(socket_path_str, 120, bollard::API_DEFAULT_VERSION) - .map_err(|err| { - Error::execution(format!("failed to create Docker client: {err}")) - })?; - let version = docker.version().await.map_err(|err| { - Error::execution(format!("failed to query Docker daemon version: {err}")) - })?; - let info = docker.info().await.map_err(|err| { - Error::execution(format!("failed to query Docker daemon info: {err}")) - })?; - let supports_gpu = info - .cdi_spec_dirs - .as_ref() - .is_some_and(|dirs| !dirs.is_empty()); - let cdi_gpu_inventory = docker_cdi_gpu_inventory(&info); - let allow_all_default_gpu = docker_info_reports_wsl2(&info); - validate_sandbox_pids_limit(docker_config.sandbox_pids_limit)?; - let gateway_port = config.bind_address.port(); - if gateway_port == 0 { - return Err(Error::config( - "docker compute driver requires a fixed non-zero gateway bind port", - )); - } - let network_name = docker_network_name(docker_config); - let bridge_gateway_ip = ensure_bridge_network(&docker, &network_name).await?; - let host_gateway_ip = parse_optional_host_gateway_ip(&docker_config.host_gateway_ip)?; - let gateway_route = - docker_gateway_route(&info, bridge_gateway_ip, gateway_port, host_gateway_ip); - let mut docker_config = docker_config.clone(); - if docker_config.grpc_endpoint.trim().is_empty() { - let scheme = if docker_guest_tls_configured(&docker_config) { - "https" - } else { - "http" - }; - docker_config.grpc_endpoint = - format!("{scheme}://{HOST_OPENSHELL_INTERNAL}:{gateway_port}"); - } - let grpc_endpoint = docker_container_openshell_endpoint( - &docker_config.grpc_endpoint, - HOST_OPENSHELL_INTERNAL, - gateway_port, - ); - let daemon_arch = normalize_docker_arch(version.arch.as_deref().unwrap_or_default()); - let supervisor_bin = resolve_supervisor_bin(&docker, &docker_config, &daemon_arch).await?; - let guest_tls = docker_guest_tls_paths(&docker_config)?; - - let driver = Self { - docker: Arc::new(docker), - config: DockerDriverRuntimeConfig { - default_image: docker_config.default_image.clone(), - image_pull_policy: docker_config.image_pull_policy.clone(), - sandbox_namespace: docker_config.sandbox_namespace.clone(), - grpc_endpoint, - network_name, - gateway_route, - ssh_socket_path: docker_config.ssh_socket_path.clone(), - stop_timeout_secs: DEFAULT_STOP_TIMEOUT_SECS, - log_level: config.log_level.clone(), - supervisor_bin, - guest_tls, - daemon_version: version.version.unwrap_or_else(|| "unknown".to_string()), - supports_gpu, - allow_all_default_gpu, - sandbox_pids_limit: docker_config.sandbox_pids_limit, - enable_bind_mounts: docker_config.enable_bind_mounts, - }, - events: broadcast::channel(WATCH_BUFFER).0, - pending: Arc::new(Mutex::new(HashMap::new())), - supervisor_readiness, - gpu_selector: Arc::new(CdiGpuDefaultSelector::new( - cdi_gpu_inventory, - allow_all_default_gpu, - )), - }; - - let poll_driver = driver.clone(); - tokio::spawn(async move { - poll_driver.poll_loop().await; - }); - - Ok(driver) - } - - #[must_use] - pub fn gateway_bind_addresses(&self) -> Vec { - match self.config.gateway_route { - DockerGatewayRoute::Bridge { bind_address, .. } => vec![bind_address], - DockerGatewayRoute::HostGateway => Vec::new(), - } - } - - fn capabilities(&self) -> GetCapabilitiesResponse { - openshell_core::driver_utils::build_capabilities_response( - "docker", - &self.config.daemon_version, - &self.config.default_image, - ) - } - - #[cfg(test)] - fn validate_sandbox( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - ) -> Result<(), Status> { - let _ = Self::validated_sandbox(sandbox, config)?; - Ok(()) - } - - fn validated_sandbox<'a>( - sandbox: &'a DriverSandbox, - config: &DockerDriverRuntimeConfig, - ) -> Result, Status> { - let spec = sandbox - .spec - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; - let template = spec - .template - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - - Self::validate_sandbox_template_base(template)?; - let _ = docker_resource_limits(template)?; - let driver_config = - DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; - validate_docker_driver_mounts(&driver_config.mounts, config.enable_bind_mounts)?; - let gpu_requirements = driver_gpu_requirements(spec.resource_requirements.as_ref()); - Self::validate_gpu_request(gpu_requirements, config.supports_gpu, &driver_config)?; - Ok(ValidatedDockerSandbox { - template, - driver_config, - gpu_requirements, - }) - } - - fn validate_sandbox_template_base(template: &DriverSandboxTemplate) -> Result<(), Status> { - if template.image.trim().is_empty() { - return Err(Status::failed_precondition( - "docker sandboxes require a template image", - )); - } - if !template.agent_socket_path.trim().is_empty() { - return Err(Status::failed_precondition( - "docker compute driver does not support template.agent_socket_path", - )); - } - if template - .platform_config - .as_ref() - .is_some_and(|config| !config.fields.is_empty()) - { - return Err(Status::failed_precondition( - "docker compute driver does not support template.platform_config", - )); - } - - Ok(()) - } - - fn validate_sandbox_auth(sandbox: &DriverSandbox) -> Result<(), Status> { - let token_present = sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.trim().is_empty()); - if token_present { - return Ok(()); - } - - Err(Status::failed_precondition( - "docker sandboxes require gateway JWT auth; configure [openshell.gateway.gateway_jwt]", - )) - } - - fn validate_gpu_request( - gpu_requirements: Option<&GpuResourceRequirements>, - supports_gpu: bool, - driver_config: &DockerSandboxDriverConfig, - ) -> Result<(), Status> { - let requested_count = - effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)?; - if requested_count.is_some() && !supports_gpu { - return Err(Status::failed_precondition( - "docker GPU sandboxes require Docker CDI support. Enable CDI on the Docker daemon, then restart the OpenShell gateway/server so GPU capability is detected.", - )); - } - - if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - } - - Ok(()) - } - - async fn validate_user_volume_mounts_available( - &self, - driver_config: &DockerSandboxDriverConfig, - ) -> Result<(), Status> { - for mount in &driver_config.mounts { - if let DockerDriverMountConfig::Volume { source, .. } = mount { - match self.docker.inspect_volume(source).await { - Ok(volume) => { - if !self.config.enable_bind_mounts && docker_volume_is_bind_backed(&volume) - { - return Err(Status::failed_precondition(format!( - "docker volume '{source}' is backed by a host bind mount and requires enable_bind_mounts = true in [openshell.drivers.docker]" - ))); - } - } - Err(err) if is_not_found_error(&err) => { - return Err(Status::failed_precondition(format!( - "docker volume '{source}' does not exist" - ))); - } - Err(err) => { - return Err(internal_status("inspect docker volume", err)); - } - } - } - } - Ok(()) - } - - async fn refresh_gpu_inventory(&self) -> Result<(), Status> { - let info = self - .docker - .info() - .await - .map_err(|err| internal_status("query Docker daemon info", err))?; - self.gpu_selector.refresh( - docker_cdi_gpu_inventory(&info), - self.config.allow_all_default_gpu, - ); - Ok(()) - } - - async fn resolve_gpu_cdi_devices( - &self, - gpu_requirements: Option<&GpuResourceRequirements>, - driver_config: &DockerSandboxDriverConfig, - select_default_devices: fn( - &CdiGpuDefaultSelector, - u32, - ) -> Result, CdiGpuSelectionError>, - ) -> Result>, Status> { - if let Some(cdi_devices) = driver_config.cdi_devices.as_deref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - return Ok(Some(cdi_devices.to_vec())); - } - - let Some(count) = - effective_driver_gpu_count(gpu_requirements).map_err(Status::invalid_argument)? - else { - return Ok(None); - }; - - self.refresh_gpu_inventory().await?; - select_default_devices(&self.gpu_selector, count) - .map(Some) - .map_err(docker_gpu_selection_status) - } - - async fn get_sandbox_snapshot( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result, Status> { - let container = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await?; - if let Some(sandbox) = container.and_then(|summary| { - sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) - }) { - return Ok(Some(sandbox)); - } - - Ok(self.pending_snapshot(sandbox_id, sandbox_name).await) - } - - async fn current_snapshots(&self) -> Result, Status> { - let containers = self.list_managed_container_summaries().await?; - let container_sandboxes = containers - .iter() - .filter_map(|summary| { - sandbox_from_container_summary(summary, self.supervisor_readiness.as_ref()) - }) - .collect::>(); - let mut by_id = self.pending_snapshot_map().await; - for sandbox in container_sandboxes { - by_id.insert(sandbox.id.clone(), sandbox); - } - let mut sandboxes = by_id.into_values().collect::>(); - sandboxes.sort_by(|left, right| left.id.cmp(&right.id)); - Ok(sandboxes) - } - - async fn create_sandbox_inner(&self, sandbox: &DriverSandbox) -> Result<(), Status> { - let validated = Self::validated_sandbox(sandbox, &self.config)?; - Self::validate_sandbox_auth(sandbox)?; - self.validate_user_volume_mounts_available(&validated.driver_config) - .await?; - let gpu_devices = self - .resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::peek_device_ids, - ) - .await?; - let _ = build_container_create_body_with_gpu_devices( - sandbox, - &self.config, - &validated.driver_config, - gpu_devices.as_deref(), - )?; - - if self - .find_managed_container_summary(&sandbox.id, &sandbox.name) - .await? - .is_some() - { - return Err(Status::already_exists("sandbox already exists")); - } - - self.reserve_pending_sandbox(sandbox).await?; - let image = sandbox_image(sandbox).unwrap_or_default(); - self.publish_docker_progress( - &sandbox.id, - "Scheduled", - format!("Docker sandbox accepted for image \"{image}\""), - HashMap::from([("image_ref".to_string(), image)]), - ); - self.publish_sandbox_snapshot(pending_sandbox_snapshot( - sandbox, - &self.config.sandbox_namespace, - provisioning_condition(), - false, - )); - - let driver = self.clone(); - let sandbox_for_task = sandbox.clone(); - let sandbox_id = sandbox.id.clone(); - let task = tokio::spawn(async move { - driver.provision_sandbox(sandbox_for_task).await; - }); - - let mut pending = self.pending.lock().await; - if let Some(record) = pending.get_mut(&sandbox_id) { - record.task = Some(task); - } else { - task.abort(); - } - - Ok(()) - } - - async fn provision_sandbox(&self, sandbox: DriverSandbox) { - match self.provision_sandbox_inner(&sandbox).await { - Ok(()) => { - self.clear_pending_sandbox(&sandbox.id).await; - } - Err(failure) => { - self.fail_pending_sandbox(&sandbox, &failure).await; - } - } - } - - async fn provision_sandbox_inner( - &self, - sandbox: &DriverSandbox, - ) -> Result<(), DockerProvisioningFailure> { - let validated = Self::validated_sandbox(sandbox, &self.config).map_err(|status| { - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - let template = validated.template; - self.ensure_image_available(&sandbox.id, &template.image) - .await - .map_err(|status| { - DockerProvisioningFailure::new("ImagePullFailed", status.message()) - })?; - let token_file_created = write_sandbox_token_file(sandbox, &self.config) - .await - .map_err(|status| { - DockerProvisioningFailure::new("SandboxTokenWriteFailed", status.message()) - })?; - - let container_name = container_name_for_sandbox(sandbox); - let gpu_devices = self - .resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::next_device_ids, - ) - .await - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - let create_body = build_container_create_body_with_gpu_devices( - sandbox, - &self.config, - &validated.driver_config, - gpu_devices.as_deref(), - ) - .map_err(|status| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::new("ContainerCreateFailed", status.message()) - })?; - self.docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - create_body, - ) - .await - .map_err(|err| { - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - DockerProvisioningFailure::from_status( - "ContainerCreateFailed", - create_status_from_docker_error("create docker sandbox container", err), - ) - })?; - self.publish_docker_progress( - &sandbox.id, - "Created", - format!("Created Docker container \"{container_name}\""), - HashMap::from([("container_name".to_string(), container_name.clone())]), - ); - - if let Err(err) = self.docker.start_container(&container_name, None).await { - let cleanup = self - .docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await; - if let Err(cleanup_err) = cleanup { - warn!( - sandbox_id = %sandbox.id, - container_name, - error = %cleanup_err, - "Failed to clean up Docker container after start failure" - ); - } - if token_file_created { - cleanup_sandbox_token_file(sandbox, &self.config); - } - return Err(DockerProvisioningFailure::from_status( - "ContainerStartFailed", - create_status_from_docker_error("start docker sandbox container", err), - )); - } - self.publish_docker_progress( - &sandbox.id, - "Started", - format!("Started Docker container \"{container_name}\""), - HashMap::from([("container_name".to_string(), container_name)]), - ); - if let Err(err) = self - .publish_container_snapshot(&sandbox.id, &sandbox.name) - .await - { - warn!( - sandbox_id = %sandbox.id, - error = %err, - "Failed to publish Docker sandbox snapshot after start" - ); - } - - Ok(()) - } - - async fn delete_sandbox_inner( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result { - let pending = self.remove_pending_sandbox(sandbox_id, sandbox_name).await; - if let Some(record) = pending.as_ref() - && let Some(task) = record.task.as_ref() - { - task.abort(); - } - - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - if let Some(record) = pending { - let container_name = container_name_for_sandbox(&record.sandbox); - match self - .docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - Ok(()) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); - return Ok(true); - } - Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file(&record.sandbox, &self.config); - return Ok(true); - } - Err(err) => { - return Err(internal_status("delete docker sandbox container", err)); - } - } - } - return Ok(false); - }; - let Some(target) = summary_container_target(&container) else { - return Ok(pending.is_some()); - }; - - match self - .docker - .remove_container( - &target, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - Ok(()) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); - Ok(true) - } - Err(err) if is_not_found_error(&err) => { - cleanup_sandbox_token_file_for_delete(sandbox_id, pending.as_ref(), &self.config); - Ok(pending.is_some()) - } - Err(err) => Err(internal_status("delete docker sandbox container", err)), - } - } - - async fn stop_sandbox_inner(&self, sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - if let Some(record) = self.remove_pending_sandbox(sandbox_id, sandbox_name).await { - if let Some(task) = record.task { - task.abort(); - } - cleanup_sandbox_token_file(&record.sandbox, &self.config); - self.publish_deleted(record.sandbox.id); - return Ok(()); - } - return Err(Status::not_found("sandbox not found")); - }; - let Some(target) = summary_container_target(&container) else { - return Err(Status::not_found("sandbox container has no id or name")); - }; - - match self - .docker - .stop_container( - &target, - Some( - StopContainerOptionsBuilder::default() - .t(docker_stop_timeout_secs(self.config.stop_timeout_secs)) - .build(), - ), - ) - .await - { - Ok(()) => Ok(()), - Err(err) if is_not_modified_error(&err) => Ok(()), - Err(err) if is_not_found_error(&err) => Err(Status::not_found("sandbox not found")), - Err(err) => Err(internal_status("stop docker sandbox container", err)), - } - } - - /// Start a managed sandbox container that was previously stopped. Used - /// by the gateway to resume sandboxes after a restart so that running - /// state in the gateway store is matched by an actually-running - /// container. - /// - /// Returns `Ok(true)` when a container existed and was started (or was - /// already running), `Ok(false)` when no managed container is found for - /// the sandbox, and `Err(...)` for any Docker failure. - pub async fn resume_sandbox( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result { - let Some(container) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - else { - return Ok(false); - }; - let Some(target) = summary_container_target(&container) else { - return Ok(false); - }; - let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - if !container_state_needs_resume(state) { - return Ok(true); - } - - match self.docker.start_container(&target, None).await { - Ok(()) => Ok(true), - // Already running — race with another resume path or the - // restart policy. Treat as success. - Err(err) if is_not_modified_error(&err) => Ok(true), - Err(err) if is_not_found_error(&err) => Ok(false), - Err(err) => Err(internal_status("start docker sandbox container", err)), - } - } - - pub async fn stop_managed_containers_on_shutdown(&self) -> Result { - let containers = self.list_managed_container_summaries().await?; - let targets = containers - .into_iter() - .filter_map(|container| { - let state = container.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - if container_state_needs_shutdown_stop(state) { - summary_container_target(&container) - } else { - None - } - }) - .collect::>(); - let target_count = targets.len(); - let mut stopped = 0usize; - let mut failures = Vec::new(); - let stop_timeout_secs = self.config.stop_timeout_secs; - - let mut stop_results = futures::stream::iter(targets.into_iter().map(|target| { - let docker = self.docker.clone(); - async move { - let result = docker - .stop_container( - &target, - Some( - StopContainerOptionsBuilder::default() - .t(docker_stop_timeout_secs(stop_timeout_secs)) - .build(), - ), - ) - .await; - (target, result) - } - })) - .buffer_unordered(16); - - while let Some((target, result)) = stop_results.next().await { - match result { - Ok(()) => { - stopped += 1; - } - Err(err) if is_not_found_error(&err) || is_not_modified_error(&err) => {} - Err(err) => { - warn!( - container = %target, - error = %err, - "Failed to stop Docker sandbox container during shutdown" - ); - failures.push(target); - } - } - } - - if !failures.is_empty() { - return Err(Status::internal(format!( - "failed to stop {} of {target_count} Docker sandbox containers during shutdown", - failures.len() - ))); - } - - Ok(stopped) - } - - async fn reserve_pending_sandbox(&self, sandbox: &DriverSandbox) -> Result<(), Status> { - let mut pending = self.pending.lock().await; - if pending - .values() - .any(|record| record.sandbox.id == sandbox.id || record.sandbox.name == sandbox.name) - { - return Err(Status::already_exists("sandbox already exists")); - } - - pending.insert( - sandbox.id.clone(), - PendingSandboxRecord { - sandbox: pending_sandbox_snapshot( - sandbox, - &self.config.sandbox_namespace, - provisioning_condition(), - false, - ), - task: None, - }, - ); - Ok(()) - } - - async fn pending_snapshot( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Option { - let pending = self.pending.lock().await; - pending - .values() - .find(|record| pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name)) - .map(|record| record.sandbox.clone()) - } - - async fn pending_snapshot_map(&self) -> HashMap { - let pending = self.pending.lock().await; - pending - .iter() - .map(|(sandbox_id, record)| (sandbox_id.clone(), record.sandbox.clone())) - .collect() - } - - async fn clear_pending_sandbox(&self, sandbox_id: &str) { - let mut pending = self.pending.lock().await; - pending.remove(sandbox_id); - } - - async fn remove_pending_sandbox( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Option { - let mut pending = self.pending.lock().await; - let id = pending.iter().find_map(|(id, record)| { - pending_sandbox_matches(&record.sandbox, sandbox_id, sandbox_name).then(|| id.clone()) - })?; - pending.remove(&id) - } - - async fn fail_pending_sandbox( - &self, - sandbox: &DriverSandbox, - failure: &DockerProvisioningFailure, - ) { - cleanup_sandbox_token_file(sandbox, &self.config); - let snapshot = pending_sandbox_snapshot( - sandbox, - &self.config.sandbox_namespace, - error_condition(failure.reason, &failure.message), - false, - ); - { - let mut pending = self.pending.lock().await; - if let Some(record) = pending.get_mut(&sandbox.id) { - record.sandbox = snapshot.clone(); - record.task = None; - } else { - return; - } - } - - self.publish_platform_event( - sandbox.id.clone(), - platform_event( - "docker", - "Warning", - failure.reason, - format!("Docker sandbox provisioning failed: {}", failure.message), - ), - ); - self.publish_sandbox_snapshot(snapshot); - } - - async fn publish_container_snapshot( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result<(), Status> { - if let Some(summary) = self - .find_managed_container_summary(sandbox_id, sandbox_name) - .await? - && let Some(sandbox) = - sandbox_from_container_summary(&summary, self.supervisor_readiness.as_ref()) - { - self.publish_sandbox_snapshot(sandbox); - } - Ok(()) - } - - fn publish_sandbox_snapshot(&self, sandbox: DriverSandbox) { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { - sandbox: Some(sandbox), - }, - )), - }); - } - - fn publish_deleted(&self, sandbox_id: String) { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { sandbox_id }, - )), - }); - } - - fn publish_platform_event(&self, sandbox_id: String, event: DriverPlatformEvent) { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::PlatformEvent( - WatchSandboxesPlatformEvent { - sandbox_id, - event: Some(event), - }, - )), - }); - } - - fn publish_docker_progress( - &self, - sandbox_id: &str, - reason: &str, - message: String, - mut metadata: HashMap, - ) { - attach_docker_progress_metadata(&mut metadata, reason, &message); - self.publish_platform_event( - sandbox_id.to_string(), - DriverPlatformEvent { - timestamp_ms: openshell_core::time::now_ms(), - source: "docker".to_string(), - r#type: "Normal".to_string(), - reason: reason.to_string(), - message, - metadata, - }, - ); - } - - async fn poll_loop(self) { - let mut previous = match self.current_snapshot_map().await { - Ok(snapshots) => snapshots, - Err(err) => { - warn!(error = %err, "Failed to seed Docker sandbox watch state"); - HashMap::new() - } - }; - - // Exponential backoff on consecutive Docker failures to avoid a 2s - // warn-log flood when the daemon is unreachable for an extended - // period (e.g. restart, socket removed). - let mut backoff = WATCH_POLL_INTERVAL; - loop { - tokio::time::sleep(backoff).await; - match self.current_snapshot_map().await { - Ok(current) => { - emit_snapshot_diff(&self.events, &previous, ¤t); - previous = current; - backoff = WATCH_POLL_INTERVAL; - } - Err(err) => { - warn!( - error = %err, - backoff_secs = backoff.as_secs(), - "Failed to poll Docker sandboxes" - ); - backoff = (backoff * 2).min(WATCH_POLL_MAX_BACKOFF); - } - } - } - } - - async fn current_snapshot_map(&self) -> Result, Status> { - self.current_snapshots().await.map(|snapshots| { - snapshots - .into_iter() - .map(|sandbox| (sandbox.id.clone(), sandbox)) - .collect() - }) - } - - async fn list_managed_container_summaries(&self) -> Result, Status> { - let filters = managed_container_label_filters(&self.config.sandbox_namespace, []); - self.docker - .list_containers(Some( - ListContainersOptionsBuilder::default() - .all(true) - .filters(&filters) - .build(), - )) - .await - .map_err(|err| internal_status("list Docker sandbox containers", err)) - } - - async fn find_managed_container_summary( - &self, - sandbox_id: &str, - sandbox_name: &str, - ) -> Result, Status> { - let mut label_filter_values = Vec::new(); - if !sandbox_id.is_empty() { - label_filter_values.push(format!("{LABEL_SANDBOX_ID}={sandbox_id}")); - } else if !sandbox_name.is_empty() { - label_filter_values.push(format!("{LABEL_SANDBOX_NAME}={sandbox_name}")); - } - - let filters = - managed_container_label_filters(&self.config.sandbox_namespace, label_filter_values); - let containers = self - .docker - .list_containers(Some( - ListContainersOptionsBuilder::default() - .all(true) - .filters(&filters) - .build(), - )) - .await - .map_err(|err| internal_status("find Docker sandbox container", err))?; - - Ok(containers.into_iter().find(|summary| { - let Some(labels) = summary.labels.as_ref() else { - return false; - }; - let namespace_matches = labels - .get(LABEL_SANDBOX_NAMESPACE) - .is_some_and(|value| value == &self.config.sandbox_namespace); - let id_matches = sandbox_id.is_empty() - || labels - .get(LABEL_SANDBOX_ID) - .is_some_and(|value| value == sandbox_id); - let name_matches = sandbox_name.is_empty() - || labels - .get(LABEL_SANDBOX_NAME) - .is_some_and(|value| value == sandbox_name); - namespace_matches && id_matches && name_matches - })) - } - - async fn ensure_image_available(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { - let policy = self.config.image_pull_policy.trim().to_ascii_lowercase(); - match policy.as_str() { - "" | "ifnotpresent" => { - if self.docker.inspect_image(image).await.is_ok() { - self.publish_docker_progress( - sandbox_id, - "ImagePresent", - format!("Docker image \"{image}\" is already present"), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - return Ok(()); - } - self.pull_image(sandbox_id, image).await - } - "always" => self.pull_image(sandbox_id, image).await, - "never" => match self.docker.inspect_image(image).await { - Ok(_) => { - self.publish_docker_progress( - sandbox_id, - "ImagePresent", - format!("Docker image \"{image}\" is already present"), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - Ok(()) - } - Err(err) if is_not_found_error(&err) => Err(Status::failed_precondition(format!( - "docker image '{image}' is not present locally and image_pull_policy=Never" - ))), - Err(err) => Err(internal_status("inspect Docker image", err)), - }, - other => Err(Status::failed_precondition(format!( - "unsupported docker image_pull_policy '{other}'; expected Always, IfNotPresent, or Never", - ))), - } - } - - async fn pull_image(&self, sandbox_id: &str, image: &str) -> Result<(), Status> { - self.publish_docker_progress( - sandbox_id, - "Pulling", - format!("Pulling Docker image \"{image}\""), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - let mut stream = self.docker.create_image( - Some(CreateImageOptions { - from_image: Some(image.to_string()), - ..Default::default() - }), - None, - None, - ); - while let Some(result) = stream.next().await { - let info = result.map_err(|err| internal_status("pull Docker image", err))?; - if let Some(message) = info - .error_detail - .as_ref() - .and_then(|detail| detail.message.as_ref()) - { - return Err(Status::failed_precondition(format!( - "pull Docker image '{image}' failed: {message}" - ))); - } - if let Some(event) = docker_pull_progress_event(image, &info) { - self.publish_platform_event(sandbox_id.to_string(), event); - } - } - self.publish_docker_progress( - sandbox_id, - "Pulled", - format!("Pulled Docker image \"{image}\""), - HashMap::from([("image_ref".to_string(), image.to_string())]), - ); - Ok(()) - } -} - -#[tonic::async_trait] -impl ComputeDriver for DockerComputeDriver { - type WatchSandboxesStream = WatchStream; - - async fn get_capabilities( - &self, - _request: Request, - ) -> Result, Status> { - Ok(Response::new(self.capabilities())) - } - - async fn validate_sandbox_create( - &self, - request: Request, - ) -> Result, Status> { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - let validated = Self::validated_sandbox(&sandbox, &self.config)?; - self.validate_user_volume_mounts_available(&validated.driver_config) - .await?; - let _ = self - .resolve_gpu_cdi_devices( - validated.gpu_requirements, - &validated.driver_config, - CdiGpuDefaultSelector::peek_device_ids, - ) - .await?; - Ok(Response::new(ValidateSandboxCreateResponse {})) - } - - async fn get_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - - let sandbox = self - .get_sandbox_snapshot(&request.sandbox_id, &request.sandbox_name) - .await? - .ok_or_else(|| Status::not_found("sandbox not found"))?; - - if !request.sandbox_id.is_empty() && request.sandbox_id != sandbox.id { - return Err(Status::failed_precondition( - "sandbox_id did not match the fetched sandbox", - )); - } - - Ok(Response::new(GetSandboxResponse { - sandbox: Some(sandbox), - })) - } - - async fn list_sandboxes( - &self, - _request: Request, - ) -> Result, Status> { - Ok(Response::new(ListSandboxesResponse { - sandboxes: self.current_snapshots().await?, - })) - } - - async fn create_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let sandbox = request - .into_inner() - .sandbox - .ok_or_else(|| Status::invalid_argument("sandbox is required"))?; - self.create_sandbox_inner(&sandbox).await?; - Ok(Response::new(CreateSandboxResponse {})) - } - - async fn stop_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - - self.stop_sandbox_inner(&request.sandbox_id, &request.sandbox_name) - .await?; - Ok(Response::new(StopSandboxResponse {})) - } - - async fn delete_sandbox( - &self, - request: Request, - ) -> Result, Status> { - let request = request.into_inner(); - require_sandbox_identifier(&request.sandbox_id, &request.sandbox_name)?; - - let event_sandbox_id = request.sandbox_id.clone(); - let deleted = self - .delete_sandbox_inner(&request.sandbox_id, &request.sandbox_name) - .await?; - if deleted && !event_sandbox_id.is_empty() { - let _ = self.events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { - sandbox_id: event_sandbox_id, - }, - )), - }); - } - - Ok(Response::new(DeleteSandboxResponse { deleted })) - } - - async fn watch_sandboxes( - &self, - _request: Request, - ) -> Result, Status> { - // Subscribe before taking the initial snapshot so any event emitted - // between the snapshot and this subscriber becoming active is still - // delivered. Downstream consumers treat sandbox events as - // idempotent (keyed by sandbox id), so a duplicate event is benign - // while a missed one leaks state. - let mut rx = self.events.subscribe(); - let initial = self.current_snapshots().await?; - let (tx, out_rx) = mpsc::channel(WATCH_BUFFER); - tokio::spawn(async move { - for sandbox in initial { - if tx - .send(Ok(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { - sandbox: Some(sandbox), - }, - )), - })) - .await - .is_err() - { - return; - } - } - - loop { - match rx.recv().await { - Ok(event) => { - if tx.send(Ok(event)).await.is_err() { - return; - } - } - Err(broadcast::error::RecvError::Lagged(_)) => {} - Err(broadcast::error::RecvError::Closed) => return, - } - } - }); - - Ok(Response::new(Box::pin(ReceiverStream::new(out_rx)))) - } -} - -impl DockerProvisioningFailure { - fn new(reason: &'static str, message: impl Into) -> Self { - Self { - reason, - message: message.into(), - } - } - - fn from_status(reason: &'static str, status: Status) -> Self { - Self::new(reason, status.message()) - } -} - -fn sandbox_image(sandbox: &DriverSandbox) -> Option { - sandbox - .spec - .as_ref() - .and_then(|spec| spec.template.as_ref()) - .map(|template| template.image.clone()) - .filter(|image| !image.trim().is_empty()) -} - -fn pending_sandbox_snapshot( - sandbox: &DriverSandbox, - namespace: &str, - condition: DriverCondition, - deleting: bool, -) -> DriverSandbox { - DriverSandbox { - id: sandbox.id.clone(), - name: sandbox.name.clone(), - namespace: namespace.to_string(), - spec: None, - status: Some(DriverSandboxStatus { - sandbox_name: sandbox.name.clone(), - instance_id: String::new(), - agent_fd: String::new(), - sandbox_fd: String::new(), - conditions: vec![condition], - deleting, - }), - workspace: sandbox.workspace.clone(), - } -} - -fn pending_sandbox_matches(sandbox: &DriverSandbox, sandbox_id: &str, sandbox_name: &str) -> bool { - (!sandbox_id.is_empty() && sandbox.id == sandbox_id) - || (!sandbox_name.is_empty() && sandbox.name == sandbox_name) -} - -fn provisioning_condition() -> DriverCondition { - DriverCondition { - r#type: "Ready".to_string(), - status: "False".to_string(), - reason: "Starting".to_string(), - message: "Docker container is starting".to_string(), - last_transition_time: String::new(), - } -} - -fn error_condition(reason: &str, message: &str) -> DriverCondition { - DriverCondition { - r#type: "Ready".to_string(), - status: "False".to_string(), - reason: reason.to_string(), - message: message.to_string(), - last_transition_time: String::new(), - } -} - -fn platform_event( - source: &str, - event_type: &str, - reason: &str, - message: String, -) -> DriverPlatformEvent { - DriverPlatformEvent { - timestamp_ms: openshell_core::time::now_ms(), - source: source.to_string(), - r#type: event_type.to_string(), - reason: reason.to_string(), - message, - metadata: HashMap::new(), - } -} - -fn docker_pull_progress_event(image: &str, info: &CreateImageInfo) -> Option { - let status = info.status.as_deref().map(str::trim)?; - if status.is_empty() { - return None; - } - - let mut metadata = HashMap::from([ - ("image_ref".to_string(), image.to_string()), - ("docker_status".to_string(), status.to_string()), - ]); - if let Some(layer_id) = info.id.as_deref().filter(|id| !id.is_empty()) { - metadata.insert("layer_id".to_string(), layer_id.to_string()); - } - if let Some(detail) = docker_pull_progress_detail(info) { - metadata.insert("detail".to_string(), detail); - } - attach_docker_progress_metadata(&mut metadata, "PullingLayer", status); - - Some(DriverPlatformEvent { - timestamp_ms: openshell_core::time::now_ms(), - source: "docker".to_string(), - r#type: "Normal".to_string(), - reason: "PullingLayer".to_string(), - message: docker_pull_message(info, status), - metadata, - }) -} - -fn docker_pull_message(info: &CreateImageInfo, status: &str) -> String { - info.id.as_deref().filter(|id| !id.is_empty()).map_or_else( - || format!("Docker image pull: {status}"), - |layer_id| format!("Docker image pull {layer_id}: {status}"), - ) -} - -fn docker_pull_progress_detail(info: &CreateImageInfo) -> Option { - let status = info.status.as_deref().unwrap_or("Pulling"); - let layer_id = info.id.as_deref().filter(|id| !id.is_empty()); - let progress = info - .progress_detail - .as_ref() - .and_then(format_progress_detail); - - match (layer_id, progress) { - (Some(layer_id), Some(progress)) => Some(format!("{status} {layer_id} ({progress})")), - (Some(layer_id), None) => Some(format!("{status} {layer_id}")), - (None, Some(progress)) => Some(format!("{status} ({progress})")), - (None, None) => (!status.is_empty()).then(|| status.to_string()), - } -} - -fn format_progress_detail(progress: &ProgressDetail) -> Option { - let current = progress.current.and_then(|value| u64::try_from(value).ok()); - let total = progress - .total - .and_then(|value| u64::try_from(value).ok()) - .filter(|value| *value > 0); - - match (current, total) { - (Some(current), Some(total)) => { - Some(format!("{}/{}", format_bytes(current), format_bytes(total))) - } - (Some(current), _) if current > 0 => Some(format_bytes(current)), - _ => None, - } -} - -fn attach_docker_progress_metadata( - metadata: &mut HashMap, - reason: &str, - message: &str, -) { - match reason { - "Scheduled" => { - mark_progress_complete( - metadata, - PROGRESS_STEP_REQUESTING_SANDBOX, - "Sandbox allocated", - ); - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(image) = metadata.get("image_ref").cloned() { - mark_progress_detail(metadata, image); - } - } - "Pulling" => { - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(image) = metadata.get("image_ref").cloned() { - mark_progress_detail(metadata, image); - } - } - "PullingLayer" => { - mark_progress_active(metadata, PROGRESS_STEP_PULLING_IMAGE); - if let Some(detail) = metadata - .get("detail") - .cloned() - .filter(|detail| !detail.is_empty()) - { - mark_progress_detail(metadata, detail); - } else if !message.is_empty() { - mark_progress_detail(metadata, message); - } - } - "ImagePresent" => { - mark_progress_complete( - metadata, - PROGRESS_STEP_PULLING_IMAGE, - "Image already present", - ); - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - } - "Pulled" => { - mark_progress_complete(metadata, PROGRESS_STEP_PULLING_IMAGE, "Image pulled"); - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - } - "Created" => { - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - mark_progress_detail(metadata, "Container created"); - } - "Started" => { - mark_progress_active(metadata, PROGRESS_STEP_STARTING_SANDBOX); - mark_progress_detail(metadata, "Waiting for supervisor relay"); - } - _ => {} - } -} - -#[cfg(test)] -fn docker_driver_config( - template: &DriverSandboxTemplate, - enable_bind_mounts: bool, -) -> Result { - let config = - DockerSandboxDriverConfig::from_template(template).map_err(Status::invalid_argument)?; - validate_docker_driver_mounts(&config.mounts, enable_bind_mounts)?; - Ok(config) -} - -/// Collect user-supplied bind mounts as string-format binds. -/// -/// Bind mounts use the legacy `Binds` field (`-v` syntax) rather than the -/// structured `Mount` API because the Docker Engine Mount object does not -/// support `SELinux` relabelling (`:z` / `:Z`). The string format does. -fn docker_driver_bind_strings(config: &DockerSandboxDriverConfig) -> Result, Status> { - config - .mounts - .iter() - .filter_map(|m| match m { - DockerDriverMountConfig::Bind { - source, - target, - read_only, - selinux_label, - } => Some(docker_bind_string( - source, - target, - *read_only, - *selinux_label, - )), - _ => None, - }) - .collect() -} - -fn docker_bind_string( - source: &str, - target: &str, - read_only: bool, - selinux_label: Option, -) -> Result { - driver_mounts::validate_absolute_mount_source(source, "bind source") - .map_err(Status::failed_precondition)?; - // Legacy `-v` binds silently create missing source directories as empty, - // root-owned paths. The structured `--mount` API that was used before this - // change rejected missing sources at container-create time. Preserve that - // fail-fast behaviour with an explicit existence check. - if !Path::new(source).exists() { - return Err(Status::failed_precondition(format!( - "bind source path does not exist: {source}" - ))); - } - driver_mounts::validate_container_mount_target(target).map_err(Status::failed_precondition)?; - let normalized_target = driver_mounts::normalize_mount_target(target); - - let mut opts = Vec::new(); - if read_only { - opts.push("ro"); - } - match selinux_label { - Some(SelinuxLabel::Shared) => opts.push("z"), - Some(SelinuxLabel::Private) => opts.push("Z"), - None => {} - } - - if opts.is_empty() { - Ok(format!("{source}:{normalized_target}")) - } else { - Ok(format!("{source}:{normalized_target}:{}", opts.join(","))) - } -} - -/// Collect user-supplied non-bind mounts as structured `Mount` objects. -fn docker_driver_mounts(config: &DockerSandboxDriverConfig) -> Result, Status> { - config - .mounts - .iter() - .filter_map(|m| docker_mount_from_config(m).transpose()) - .collect() -} - -fn docker_mount_from_config(config: &DockerDriverMountConfig) -> Result, Status> { - match config { - DockerDriverMountConfig::Bind { .. } => { - // Bind mounts are handled via docker_driver_bind_strings. - Ok(None) - } - DockerDriverMountConfig::Volume { - source, - target, - read_only, - subpath, - } => Ok(Some(Mount { - typ: Some(MountTypeEnum::VOLUME), - source: Some(source.clone()), - target: Some(target.clone()), - read_only: Some(*read_only), - volume_options: subpath.as_ref().map(|subpath| MountVolumeOptions { - subpath: Some(subpath.clone()), - ..Default::default() - }), - ..Default::default() - })), - DockerDriverMountConfig::Tmpfs { - target, - options, - size_bytes, - mode, - } => Ok(Some(Mount { - typ: Some(MountTypeEnum::TMPFS), - target: Some(target.clone()), - tmpfs_options: Some(MountTmpfsOptions { - size_bytes: validate_optional_positive_integral_i64( - *size_bytes, - "tmpfs size_bytes", - )?, - mode: validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?, - options: (!options.is_empty()) - .then(|| { - options - .iter() - .map(|option| docker_tmpfs_option(option)) - .collect::, _>>() - }) - .transpose()?, - }), - ..Default::default() - })), - DockerDriverMountConfig::Image { .. } => Err(Status::failed_precondition( - "invalid docker driver_config: docker image mounts are not supported", - )), - } -} - -fn validate_docker_driver_mounts( - mounts: &[DockerDriverMountConfig], - enable_bind_mounts: bool, -) -> Result<(), Status> { - let mut targets = HashSet::new(); - for mount in mounts { - let target = match mount { - DockerDriverMountConfig::Bind { source, target, .. } => { - if !enable_bind_mounts { - return Err(Status::failed_precondition( - "docker bind mounts require enable_bind_mounts = true in [openshell.drivers.docker]", - )); - } - driver_mounts::validate_absolute_mount_source(source, "bind source") - .map_err(Status::failed_precondition)?; - target - } - DockerDriverMountConfig::Volume { - source, - target, - subpath, - .. - } => { - driver_mounts::validate_mount_source(source, "volume source") - .map_err(Status::failed_precondition)?; - if let Some(subpath) = subpath { - driver_mounts::validate_mount_subpath(subpath) - .map_err(Status::failed_precondition)?; - } - target - } - DockerDriverMountConfig::Tmpfs { - target, - options, - size_bytes, - mode, - } => { - validate_optional_positive_integral_i64(*size_bytes, "tmpfs size_bytes")?; - validate_optional_nonnegative_integral_i64(*mode, "tmpfs mode")?; - for option in options { - docker_tmpfs_option(option)?; - } - target - } - DockerDriverMountConfig::Image { - source, - target, - read_only, - subpath, - } => { - let _ = (source, target, read_only, subpath); - return Err(Status::failed_precondition( - "invalid docker driver_config: docker image mounts are not supported", - )); - } - }; - driver_mounts::validate_container_mount_target(target) - .map_err(Status::failed_precondition)?; - let normalized_target = driver_mounts::normalize_mount_target(target); - if !targets.insert(normalized_target.clone()) { - return Err(Status::failed_precondition(format!( - "duplicate docker driver_config mount target '{normalized_target}'" - ))); - } - } - Ok(()) -} - -fn validate_optional_positive_integral_i64( - value: Option, - field: &str, -) -> Result, Status> { - let Some(value) = validate_optional_integral_i64(value, field)? else { - return Ok(None); - }; - if value <= 0 { - return Err(Status::failed_precondition(format!( - "{field} must be positive" - ))); - } - Ok(Some(value)) -} - -fn validate_optional_nonnegative_integral_i64( - value: Option, - field: &str, -) -> Result, Status> { - let Some(value) = validate_optional_integral_i64(value, field)? else { - return Ok(None); - }; - if value < 0 { - return Err(Status::failed_precondition(format!( - "{field} must be zero or greater" - ))); - } - Ok(Some(value)) -} - -fn validate_optional_integral_i64(value: Option, field: &str) -> Result, Status> { - let Some(value) = value else { - return Ok(None); - }; - if !value.is_finite() || value.fract() != 0.0 { - return Err(Status::failed_precondition(format!( - "{field} must be an integer" - ))); - } - value.to_string().parse::().map(Some).map_err(|_| { - Status::failed_precondition(format!("{field} must be representable as an i64")) - }) -} - -fn docker_tmpfs_option(option: &str) -> Result, Status> { - let option = option.trim(); - if option.is_empty() { - return Err(Status::failed_precondition( - "tmpfs options must not contain empty values", - )); - } - if let Some((key, value)) = option.split_once('=') { - let key = key.trim(); - let value = value.trim(); - if key.is_empty() || value.is_empty() { - return Err(Status::failed_precondition( - "tmpfs key=value options must include both key and value", - )); - } - Ok(vec![key.to_string(), value.to_string()]) - } else { - Ok(vec![option.to_string()]) - } -} - -fn docker_volume_is_bind_backed(volume: &bollard::models::Volume) -> bool { - volume.driver == "local" - && volume.options.get("o").is_some_and(|options| { - options.split(',').any(|option| { - let option = option.trim(); - option.eq_ignore_ascii_case("bind") || option.eq_ignore_ascii_case("rbind") - }) - }) -} - -fn build_binds( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result, Status> { - let mut binds = vec![format!( - "{}:{}:ro,z", - config.supervisor_bin.display(), - SUPERVISOR_MOUNT_PATH - )]; - if let Some(tls) = &config.guest_tls { - binds.push(format!("{}:{}:ro,z", tls.ca.display(), TLS_CA_MOUNT_PATH)); - binds.push(format!( - "{}:{}:ro,z", - tls.cert.display(), - TLS_CERT_MOUNT_PATH - )); - binds.push(format!("{}:{}:ro,z", tls.key.display(), TLS_KEY_MOUNT_PATH)); - } - if sandbox - .spec - .as_ref() - .is_some_and(|spec| !spec.sandbox_token.is_empty()) - { - binds.push(format!( - "{}:{}:ro,z", - sandbox_token_host_path(sandbox, config)?.display(), - SANDBOX_TOKEN_MOUNT_PATH - )); - } - Ok(binds) -} - -fn sandbox_token_host_path( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - sandbox_token_host_path_by_id(&sandbox.id, config) -} - -fn sandbox_token_host_path_by_id( - sandbox_id: &str, - config: &DockerDriverRuntimeConfig, -) -> Result { - openshell_core::driver_utils::sandbox_token_path( - "docker-sandbox-tokens", - Some(&config.sandbox_namespace), - sandbox_id, - ) - .map_err(|err| { - Status::internal(format!( - "resolve sandbox token state directory failed: {err}" - )) - }) -} - -async fn write_sandbox_token_file( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - let Some(spec) = sandbox.spec.as_ref() else { - return Ok(false); - }; - if spec.sandbox_token.is_empty() { - return Ok(false); - } - let path = sandbox_token_host_path(sandbox, config)?; - if let Some(parent) = path.parent() { - openshell_core::paths::create_dir_restricted(parent).map_err(|err| { - Status::internal(format!( - "create sandbox token directory {} failed: {err}", - parent.display() - )) - })?; - } - tokio::fs::write(&path, format!("{}\n", spec.sandbox_token)) - .await - .map_err(|err| { - Status::internal(format!( - "write sandbox token file {} failed: {err}", - path.display() - )) - })?; - openshell_core::paths::set_file_owner_only(&path).map_err(|err| { - Status::internal(format!( - "restrict sandbox token file {} failed: {err}", - path.display() - )) - })?; - Ok(true) -} - -fn cleanup_sandbox_token_file(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) { - cleanup_sandbox_token_file_by_id(&sandbox.id, config); -} - -fn cleanup_sandbox_token_file_for_delete( - sandbox_id: &str, - pending: Option<&PendingSandboxRecord>, - config: &DockerDriverRuntimeConfig, -) { - if !sandbox_id.is_empty() { - cleanup_sandbox_token_file_by_id(sandbox_id, config); - } else if let Some(record) = pending { - cleanup_sandbox_token_file(&record.sandbox, config); - } -} - -fn cleanup_sandbox_token_file_by_id(sandbox_id: &str, config: &DockerDriverRuntimeConfig) { - let Ok(path) = sandbox_token_host_path_by_id(sandbox_id, config) else { - return; - }; - if let Err(err) = std::fs::remove_file(&path) - && err.kind() != std::io::ErrorKind::NotFound - { - warn!( - sandbox_id = %sandbox_id, - path = %path.display(), - error = %err, - "Failed to remove Docker sandbox token file" - ); - } - if let Some(dir) = path.parent() { - let _ = std::fs::remove_dir(dir); - } -} - -fn build_environment(sandbox: &DriverSandbox, config: &DockerDriverRuntimeConfig) -> Vec { - let mut environment = HashMap::from([ - ("HOME".to_string(), "/root".to_string()), - ("PATH".to_string(), SUPERVISOR_PATH.to_string()), - ("TERM".to_string(), "xterm".to_string()), - ( - "OPENSHELL_LOG_LEVEL".to_string(), - openshell_core::driver_utils::sandbox_log_level(sandbox, &config.log_level), - ), - ]); - - if let Some(spec) = sandbox.spec.as_ref() { - let mut user_env = HashMap::new(); - if let Some(template) = spec.template.as_ref() { - user_env.extend(template.environment.clone()); - } - user_env.extend(spec.environment.clone()); - environment.extend(user_env.clone()); - if !user_env.is_empty() - && let Ok(json) = serde_json::to_string(&user_env) - { - environment.insert( - openshell_core::sandbox_env::USER_ENVIRONMENT.to_string(), - json, - ); - } - } - - environment.insert( - openshell_core::sandbox_env::ENDPOINT.to_string(), - config.grpc_endpoint.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_ID.to_string(), - sandbox.id.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX.to_string(), - sandbox.name.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SSH_SOCKET_PATH.to_string(), - config.ssh_socket_path.clone(), - ); - environment.insert( - openshell_core::sandbox_env::SANDBOX_COMMAND.to_string(), - SANDBOX_COMMAND.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TELEMETRY_ENABLED.to_string(), - openshell_core::telemetry::enabled_env_value().to_string(), - ); - // The root supervisor executes namespace helpers during bootstrap; keep - // their search path driver-owned even when the template/spec set PATH. - environment.insert("PATH".to_string(), SUPERVISOR_PATH.to_string()); - if config.guest_tls.is_some() { - environment.insert( - openshell_core::sandbox_env::TLS_CA.to_string(), - TLS_CA_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_CERT.to_string(), - TLS_CERT_MOUNT_PATH.to_string(), - ); - environment.insert( - openshell_core::sandbox_env::TLS_KEY.to_string(), - TLS_KEY_MOUNT_PATH.to_string(), - ); - } - - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN); - environment.remove(openshell_core::sandbox_env::SANDBOX_TOKEN_FILE); - - // Gateway-minted sandbox JWT. Keep the raw bearer out of container - // metadata; the supervisor reads it from this driver-owned bind mount. - if let Some(spec) = sandbox.spec.as_ref() - && !spec.sandbox_token.is_empty() - { - environment.insert( - openshell_core::sandbox_env::SANDBOX_TOKEN_FILE.to_string(), - SANDBOX_TOKEN_MOUNT_PATH.to_string(), - ); - } - - let mut pairs = environment.into_iter().collect::>(); - pairs.sort_by(|left, right| left.0.cmp(&right.0)); - pairs - .into_iter() - .map(|(key, value)| format!("{key}={value}")) - .collect() -} - -fn docker_cdi_gpu_inventory(info: &SystemInfo) -> CdiGpuInventory { - CdiGpuInventory::new( - info.discovered_devices - .as_deref() - .unwrap_or_default() - .iter() - .filter(|device| device.source.as_deref() == Some("cdi")) - .filter_map(|device| device.id.as_deref()), - ) -} - -fn docker_info_reports_wsl2(info: &SystemInfo) -> bool { - [ - info.kernel_version.as_deref(), - info.operating_system.as_deref(), - ] - .into_iter() - .flatten() - .any(os_or_kernel_reports_wsl2) -} - -fn os_or_kernel_reports_wsl2(value: &str) -> bool { - let value = value.to_ascii_lowercase(); - value.contains("wsl2") || value.contains("microsoft-standard") -} - -fn docker_gpu_selection_status(err: CdiGpuSelectionError) -> Status { - Status::failed_precondition(err.to_string()) -} - -#[cfg(test)] -fn build_container_create_body( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, -) -> Result { - let template = sandbox - .spec - .as_ref() - .and_then(|spec| spec.template.as_ref()) - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - let driver_config = docker_driver_config(template, config.enable_bind_mounts)?; - let gpu_requirements = sandbox - .spec - .as_ref() - .and_then(|spec| driver_gpu_requirements(spec.resource_requirements.as_ref())); - let cdi_devices = if let Some(cdi_devices) = driver_config.cdi_devices.as_ref() { - validate_specific_gpu_device_request( - gpu_requirements, - cdi_devices, - "driver_config.cdi_devices", - ) - .map_err(Status::invalid_argument)?; - Some(cdi_devices.as_slice()) - } else { - None - }; - build_container_create_body_with_gpu_devices(sandbox, config, &driver_config, cdi_devices) -} - -fn build_container_create_body_with_gpu_devices( - sandbox: &DriverSandbox, - config: &DockerDriverRuntimeConfig, - driver_config: &DockerSandboxDriverConfig, - gpu_device_ids: Option<&[String]>, -) -> Result { - let spec = sandbox - .spec - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec is required"))?; - let template = spec - .template - .as_ref() - .ok_or_else(|| Status::invalid_argument("sandbox.spec.template is required"))?; - let resource_limits = docker_resource_limits(template)?; - let user_mounts = docker_driver_mounts(driver_config)?; - let user_bind_strings = docker_driver_bind_strings(driver_config)?; - let device_requests = gpu_device_ids.map(|device_ids| { - vec![DeviceRequest { - driver: Some("cdi".to_string()), - device_ids: Some(device_ids.to_vec()), - ..Default::default() - }] - }); - let mut labels = template.labels.clone(); - labels.insert( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - ); - labels.insert(LABEL_SANDBOX_ID.to_string(), sandbox.id.clone()); - labels.insert(LABEL_SANDBOX_NAME.to_string(), sandbox.name.clone()); - labels.insert( - LABEL_SANDBOX_WORKSPACE.to_string(), - sandbox.workspace.clone(), - ); - // The list/get/find paths filter by `config.sandbox_namespace`, so use - // the same value here. `DriverSandbox.namespace` is unset on the request - // path (the gateway elides it), and using it would produce containers - // that the driver itself cannot find afterwards. - labels.insert( - LABEL_SANDBOX_NAMESPACE.to_string(), - config.sandbox_namespace.clone(), - ); - - Ok(ContainerCreateBody { - image: Some(template.image.clone()), - user: Some("0".to_string()), - env: Some(build_environment(sandbox, config)), - entrypoint: Some(vec![SUPERVISOR_MOUNT_PATH.to_string()]), - // Clear the image CMD so Docker does not append inherited args to the - // supervisor entrypoint. - cmd: Some(Vec::new()), - labels: Some(labels), - host_config: Some(HostConfig { - nano_cpus: resource_limits.nano_cpus, - memory: resource_limits.memory_bytes, - pids_limit: docker_pids_limit(config.sandbox_pids_limit)?, - device_requests, - binds: { - let mut binds = build_binds(sandbox, config)?; - binds.extend(user_bind_strings); - Some(binds) - }, - mounts: Some(user_mounts), - restart_policy: Some(RestartPolicy { - name: Some(RestartPolicyNameEnum::UNLESS_STOPPED), - maximum_retry_count: None, - }), - cap_add: Some(vec![ - "SYS_ADMIN".to_string(), - "NET_ADMIN".to_string(), - "SYS_PTRACE".to_string(), - "SYSLOG".to_string(), - ]), - // The sandbox supervisor needs to bind-mount `/run/netns`, - // mark it shared, and create per-process network namespaces. - // Docker's default AppArmor profile (`docker-default`) denies - // these mount operations even with CAP_SYS_ADMIN, so we opt - // out of AppArmor confinement for sandbox containers. The - // sandbox enforces its own security boundary via Landlock, - // seccomp, OPA policy evaluation, and the dedicated network - // namespace it sets up for the agent — AppArmor at the - // container layer is redundant relative to those controls - // and conflicts with them in this case. - security_opt: Some(vec!["apparmor=unconfined".to_string()]), - network_mode: Some(config.network_name.clone()), - extra_hosts: Some(docker_extra_hosts(&config.gateway_route)), - ..Default::default() - }), - networking_config: Some(NetworkingConfig { - endpoints_config: Some(HashMap::from([( - config.network_name.clone(), - EndpointSettings::default(), - )])), - }), - ..Default::default() - }) -} - -/// Reject driver requests that arrive with neither a sandbox id nor a -/// sandbox name. Without this guard, downstream label filters degenerate -/// to "match every managed container in the namespace", which would let -/// `delete_sandbox`/`stop_sandbox`/`get_sandbox` pick an arbitrary -/// sandbox out of the set the driver manages. -fn require_sandbox_identifier(sandbox_id: &str, sandbox_name: &str) -> Result<(), Status> { - if sandbox_id.is_empty() && sandbox_name.is_empty() { - return Err(Status::invalid_argument( - "sandbox_id or sandbox_name is required", - )); - } - Ok(()) -} - -fn docker_container_openshell_endpoint(endpoint: &str, host: &str, port: u16) -> String { - let Ok(mut url) = Url::parse(endpoint) else { - return endpoint.to_string(); - }; - - if url.set_host(Some(host)).is_ok() && url.set_port(Some(port)).is_ok() { - return url.to_string(); - } - - endpoint.to_string() -} - -fn docker_network_name(config: &DockerComputeConfig) -> String { - let name = config.network_name.trim(); - if name.is_empty() { - return DEFAULT_DOCKER_NETWORK_NAME.to_string(); - } - name.to_string() -} - -fn parse_optional_host_gateway_ip(value: &str) -> CoreResult> { - let trimmed = value.trim(); - if trimmed.is_empty() { - return Ok(None); - } - - trimmed - .parse() - .map(Some) - .map_err(|err| Error::config(format!("invalid host_gateway_ip value '{trimmed}': {err}"))) -} - -fn docker_gateway_route( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, -) -> DockerGatewayRoute { - docker_gateway_route_for_host( - info, - bridge_gateway_ip, - port, - host_gateway_ip, - host_runtime_requires_host_gateway_alias(), - ) -} - -fn docker_gateway_route_for_host( - info: &SystemInfo, - bridge_gateway_ip: IpAddr, - port: u16, - host_gateway_ip: Option, - host_requires_host_gateway_alias: bool, -) -> DockerGatewayRoute { - if let Some(host_alias_ip) = host_gateway_ip { - return DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(host_alias_ip, port), - host_alias_ip, - }; - } - - if host_requires_host_gateway_alias || uses_host_gateway_alias(info) { - DockerGatewayRoute::HostGateway - } else { - DockerGatewayRoute::Bridge { - bind_address: SocketAddr::new(bridge_gateway_ip, port), - host_alias_ip: bridge_gateway_ip, - } - } -} - -fn host_runtime_requires_host_gateway_alias() -> bool { - cfg!(target_os = "macos") -} - -/// Detect Docker Desktop and behaviourally compatible runtimes - Colima, -/// Lima, Rancher Desktop, and `OrbStack` - that share Docker Desktop's routing -/// constraint: the bridge gateway IP is reachable from inside containers but -/// not from the `OpenShell` server process running on the host, so callbacks -/// must traverse `host-gateway`. -/// -/// Each runtime is detected via the daemon's reported OS string or hostname, -/// supplemented by labels where the runtime publishes them. -fn uses_host_gateway_alias(info: &SystemInfo) -> bool { - let operating_system = info - .operating_system - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if operating_system.contains("docker desktop") { - return true; - } - - let name = info - .name - .as_deref() - .unwrap_or_default() - .to_ascii_lowercase(); - if name.starts_with("colima") - || name.starts_with("lima-") - || name.starts_with("rancher-desktop") - || name.starts_with("orbstack") - { - return true; - } - - info.labels.as_ref().is_some_and(|labels| { - labels.iter().any(|label| { - label.starts_with("com.docker.desktop.") - || label.starts_with("dev.rancherdesktop.") - || label.starts_with("dev.orbstack.") - }) - }) -} - -fn docker_extra_hosts(route: &DockerGatewayRoute) -> Vec { - match route { - DockerGatewayRoute::Bridge { host_alias_ip, .. } => vec![ - format!("{HOST_DOCKER_INTERNAL}:{host_alias_ip}"), - format!("{HOST_OPENSHELL_INTERNAL}:{host_alias_ip}"), - ], - DockerGatewayRoute::HostGateway => vec![ - format!("{HOST_DOCKER_INTERNAL}:host-gateway"), - format!("{HOST_OPENSHELL_INTERNAL}:host-gateway"), - ], - } -} - -async fn ensure_bridge_network(docker: &Docker, network_name: &str) -> CoreResult { - match docker.inspect_network(network_name, None).await { - Ok(network) => return validate_bridge_network(network_name, &network), - Err(err) if !is_not_found_error(&err) => { - return Err(Error::execution(format!( - "failed to inspect Docker network '{network_name}': {err}" - ))); - } - Err(_) => {} - } - - docker - .create_network(NetworkCreateRequest { - name: network_name.to_string(), - driver: Some(DOCKER_NETWORK_DRIVER.to_string()), - attachable: Some(true), - labels: Some(HashMap::from([( - LABEL_MANAGED_BY.to_string(), - LABEL_MANAGED_BY_VALUE.to_string(), - )])), - ..Default::default() - }) - .await - .map(|_| ()) - .or_else(|err| { - if is_conflict_error(&err) { - Ok(()) - } else { - Err(Error::execution(format!( - "failed to create Docker network '{network_name}': {err}" - ))) - } - })?; - - let network = docker - .inspect_network(network_name, None) - .await - .map_err(|err| { - Error::execution(format!( - "failed to inspect Docker network '{network_name}' after create: {err}" - )) - })?; - validate_bridge_network(network_name, &network) -} - -fn validate_bridge_network( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - if network.driver.as_deref() != Some(DOCKER_NETWORK_DRIVER) { - return Err(Error::config(format!( - "Docker network '{network_name}' must use the '{DOCKER_NETWORK_DRIVER}' driver, found '{}'", - network.driver.as_deref().unwrap_or("unknown") - ))); - } - - docker_bridge_gateway_ip(network_name, network) -} - -fn docker_bridge_gateway_ip( - network_name: &str, - network: &bollard::models::NetworkInspect, -) -> CoreResult { - let Some(configs) = network.ipam.as_ref().and_then(|ipam| ipam.config.as_ref()) else { - return Err(Error::config(format!( - "Docker bridge network '{network_name}' does not expose IPAM gateway configuration" - ))); - }; - - for config in configs { - let Some(gateway) = config.gateway.as_deref() else { - continue; - }; - let ip = gateway.parse::().map_err(|err| { - Error::config(format!( - "Docker bridge network '{network_name}' has invalid gateway '{gateway}': {err}" - )) - })?; - if matches!(ip, IpAddr::V4(_)) { - return Ok(ip); - } - } - - Err(Error::config(format!( - "Docker bridge network '{network_name}' does not have an IPv4 IPAM gateway" - ))) -} - -fn docker_resource_limits( - template: &DriverSandboxTemplate, -) -> Result { - let Some(resources) = template.resources.as_ref() else { - return Ok(DockerResourceLimits::default()); - }; - - if !resources.cpu_request.trim().is_empty() { - return Err(Status::failed_precondition( - "docker compute driver does not support resources.requests.cpu", - )); - } - if !resources.memory_request.trim().is_empty() { - return Err(Status::failed_precondition( - "docker compute driver does not support resources.requests.memory", - )); - } - - Ok(DockerResourceLimits { - nano_cpus: parse_cpu_limit(&resources.cpu_limit)?, - memory_bytes: parse_memory_limit(&resources.memory_limit)?, - }) -} - -fn validate_sandbox_pids_limit(value: i64) -> CoreResult<()> { - if value < 0 { - return Err(Error::config( - "docker sandbox_pids_limit must be zero or greater", - )); - } - Ok(()) -} - -fn docker_pids_limit(value: i64) -> Result, Status> { - if value < 0 { - return Err(Status::failed_precondition( - "docker sandbox_pids_limit must be zero or greater", - )); - } - if value == 0 { - Ok(None) - } else { - Ok(Some(value)) - } -} - -#[allow(clippy::cast_possible_truncation)] -fn parse_cpu_limit(value: &str) -> Result, Status> { - let value = value.trim(); - if value.is_empty() { - return Ok(None); - } - if let Some(millicores) = value.strip_suffix('m') { - let millicores = millicores.parse::().map_err(|_| { - Status::failed_precondition(format!( - "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", - )) - })?; - if millicores <= 0 { - return Err(Status::failed_precondition( - "docker cpu_limit must be greater than zero", - )); - } - return Ok(Some(millicores.saturating_mul(1_000_000))); - } - - let cores = value.parse::().map_err(|_| { - Status::failed_precondition(format!( - "invalid docker cpu_limit '{value}'; expected an integer or millicore quantity", - )) - })?; - if !cores.is_finite() || cores <= 0.0 { - return Err(Status::failed_precondition( - "docker cpu_limit must be greater than zero", - )); - } - - Ok(Some((cores * 1_000_000_000.0).round() as i64)) -} - -#[allow(clippy::cast_possible_truncation)] -fn parse_memory_limit(value: &str) -> Result, Status> { - let value = value.trim(); - if value.is_empty() { - return Ok(None); - } - - let number_end = value - .find(|ch: char| !(ch.is_ascii_digit() || ch == '.')) - .unwrap_or(value.len()); - let (number, suffix) = value.split_at(number_end); - let amount = number.parse::().map_err(|_| { - Status::failed_precondition(format!( - "invalid docker memory_limit '{value}'; expected a Kubernetes-style quantity", - )) - })?; - if !amount.is_finite() || amount <= 0.0 { - return Err(Status::failed_precondition( - "docker memory_limit must be greater than zero", - )); - } - - let multiplier = match suffix { - "" => 1_f64, - "Ki" => 1024_f64, - "Mi" => 1024_f64.powi(2), - "Gi" => 1024_f64.powi(3), - "Ti" => 1024_f64.powi(4), - "Pi" => 1024_f64.powi(5), - "Ei" => 1024_f64.powi(6), - "K" => 1000_f64, - "M" => 1000_f64.powi(2), - "G" => 1000_f64.powi(3), - "T" => 1000_f64.powi(4), - "P" => 1000_f64.powi(5), - "E" => 1000_f64.powi(6), - _ => { - return Err(Status::failed_precondition(format!( - "invalid docker memory_limit suffix '{suffix}'", - ))); - } - }; - - Ok(Some((amount * multiplier).round() as i64)) -} - -fn sandbox_from_container_summary( - summary: &ContainerSummary, - readiness: &dyn SupervisorReadiness, -) -> Option { - let labels = summary.labels.as_ref()?; - let id = labels.get(LABEL_SANDBOX_ID)?.clone(); - let name = labels.get(LABEL_SANDBOX_NAME)?.clone(); - let namespace = labels - .get(LABEL_SANDBOX_NAMESPACE) - .cloned() - .unwrap_or_default(); - let workspace = labels - .get(LABEL_SANDBOX_WORKSPACE) - .cloned() - .unwrap_or_default(); - - let supervisor_connected = readiness.is_supervisor_connected(&id); - Some(DriverSandbox { - id, - name: name.clone(), - namespace, - spec: None, - status: Some(driver_status_from_summary( - summary, - &name, - supervisor_connected, - )), - workspace, - }) -} - -fn driver_status_from_summary( - summary: &ContainerSummary, - sandbox_name: &str, - supervisor_connected: bool, -) -> DriverSandboxStatus { - let state = summary.state.unwrap_or(ContainerSummaryStateEnum::EMPTY); - let (ready, reason, message, deleting) = container_ready_condition(state, supervisor_connected); - - DriverSandboxStatus { - sandbox_name: summary_container_name(summary).unwrap_or_else(|| sandbox_name.to_string()), - instance_id: summary.id.clone().unwrap_or_default(), - agent_fd: String::new(), - sandbox_fd: String::new(), - conditions: vec![DriverCondition { - r#type: "Ready".to_string(), - status: ready.to_string(), - reason: reason.to_string(), - message: message.to_string(), - last_transition_time: String::new(), - }], - deleting, - } -} - -fn container_ready_condition( - state: ContainerSummaryStateEnum, - supervisor_connected: bool, -) -> (&'static str, &'static str, &'static str, bool) { - match state { - ContainerSummaryStateEnum::RUNNING => { - if supervisor_connected { - ( - "True", - "SupervisorConnected", - "Supervisor relay is live", - false, - ) - } else { - ( - "False", - "DependenciesNotReady", - "Container is running; waiting for supervisor relay", - false, - ) - } - } - ContainerSummaryStateEnum::CREATED => ("False", "Starting", "Container created", false), - ContainerSummaryStateEnum::RESTARTING => ( - "False", - "ContainerRestarting", - "Container is restarting after a failure", - false, - ), - ContainerSummaryStateEnum::EMPTY => { - ("False", "Starting", "Container state is unknown", false) - } - ContainerSummaryStateEnum::REMOVING => { - ("False", "Deleting", "Container is being removed", true) - } - ContainerSummaryStateEnum::PAUSED => { - ("False", "ContainerPaused", "Container is paused", false) - } - ContainerSummaryStateEnum::EXITED => { - ("False", "ContainerExited", "Container exited", false) - } - ContainerSummaryStateEnum::DEAD => ("False", "ContainerDead", "Container is dead", false), - } -} - -fn summary_container_name(summary: &ContainerSummary) -> Option { - summary - .names - .as_ref() - .and_then(|names| names.first()) - .map(|name| name.trim_start_matches('/').to_string()) - .filter(|name| !name.is_empty()) -} - -fn summary_container_target(summary: &ContainerSummary) -> Option { - // Prefer the container ID: it's stable while the container exists and is - // accepted by Docker APIs just like a name. Fall back to the parsed name - // for transient summaries that do not include an ID. - summary - .id - .as_deref() - .filter(|id| !id.is_empty()) - .map(str::to_string) - .or_else(|| summary_container_name(summary)) -} - -fn container_state_needs_shutdown_stop(state: ContainerSummaryStateEnum) -> bool { - matches!( - state, - ContainerSummaryStateEnum::RUNNING - | ContainerSummaryStateEnum::RESTARTING - | ContainerSummaryStateEnum::PAUSED - ) -} - -/// States from which a managed container can be brought back to running by -/// `start_container`. Skip `Restarting` (already coming up), `Removing`, -/// `Dead` (terminal), `Paused` (needs `unpause`, not `start`), and -/// `Running` (nothing to do). -fn container_state_needs_resume(state: ContainerSummaryStateEnum) -> bool { - matches!( - state, - ContainerSummaryStateEnum::EXITED | ContainerSummaryStateEnum::CREATED - ) -} - -fn docker_stop_timeout_secs(timeout_secs: u32) -> i32 { - i32::try_from(timeout_secs).unwrap_or(i32::MAX) -} - -fn emit_snapshot_diff( - events: &broadcast::Sender, - previous: &HashMap, - current: &HashMap, -) { - for (sandbox_id, sandbox) in current { - if previous.get(sandbox_id) == Some(sandbox) { - continue; - } - let _ = events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Sandbox( - WatchSandboxesSandboxEvent { - sandbox: Some(sandbox.clone()), - }, - )), - }); - } - - for sandbox_id in previous.keys() { - if current.contains_key(sandbox_id) { - continue; - } - let _ = events.send(WatchSandboxesEvent { - payload: Some(watch_sandboxes_event::Payload::Deleted( - WatchSandboxesDeletedEvent { - sandbox_id: sandbox_id.clone(), - }, - )), - }); - } -} - -fn label_filters(values: impl IntoIterator) -> HashMap> { - HashMap::from([("label".to_string(), values.into_iter().collect())]) -} - -fn managed_container_label_filters( - sandbox_namespace: &str, - extra_values: impl IntoIterator, -) -> HashMap> { - let mut values = vec![ - format!("{LABEL_MANAGED_BY}={LABEL_MANAGED_BY_VALUE}"), - format!("{LABEL_SANDBOX_NAMESPACE}={sandbox_namespace}"), - ]; - values.extend(extra_values); - label_filters(values) -} - -/// Maximum Docker container name length. Docker's own limit is 253 bytes, but -/// we cap at a conservative 200 to leave headroom for tooling that truncates -/// names further. -const MAX_CONTAINER_NAME_LEN: usize = 200; -const CONTAINER_NAME_PREFIX: &str = "openshell-"; - -fn container_name_for_sandbox(sandbox: &DriverSandbox) -> String { - let id_suffix = sanitize_docker_name(&sandbox.id); - let workspace = sanitize_docker_name(&sandbox.workspace); - let name = sanitize_docker_name(&sandbox.name); - - // Format: openshell-{workspace}--{name}-{id} - // The workspace and id are never truncated — they ensure uniqueness. - // Only the sandbox name portion is truncated when the total exceeds - // MAX_CONTAINER_NAME_LEN. - - if name.is_empty() { - let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); - if base.len() > MAX_CONTAINER_NAME_LEN { - base.truncate(MAX_CONTAINER_NAME_LEN); - } - return trim_container_name_tail(base); - } - - // Reserve space for fixed parts: prefix + workspace + "--" + "-" + id - let reserved = CONTAINER_NAME_PREFIX.len() + workspace.len() + 2 + 1 + id_suffix.len(); - if reserved >= MAX_CONTAINER_NAME_LEN { - let mut base = format!("{CONTAINER_NAME_PREFIX}{workspace}---{id_suffix}"); - base.truncate(MAX_CONTAINER_NAME_LEN); - return trim_container_name_tail(base); - } - - let name_budget = MAX_CONTAINER_NAME_LEN - reserved; - let truncated_name = if name.len() > name_budget { - trim_container_name_tail(name[..name_budget].to_string()) - } else { - name - }; - format!("{CONTAINER_NAME_PREFIX}{workspace}--{truncated_name}-{id_suffix}") -} - -/// Docker container names may not end with `-`, `.`, or `_`. Truncation can -/// leave one of those trailing, so strip them before returning. -fn trim_container_name_tail(mut value: String) -> String { - while value - .chars() - .last() - .is_some_and(|ch| matches!(ch, '-' | '.' | '_')) - { - value.pop(); - } - value -} - -fn sanitize_docker_name(value: &str) -> String { - value - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() || matches!(ch, '_' | '.' | '-') { - ch.to_ascii_lowercase() - } else { - '-' - } - }) - .collect::() - .trim_matches('-') - .to_string() -} - -fn normalize_docker_arch(arch: &str) -> String { - match arch { - "x86_64" => "amd64".to_string(), - "aarch64" => "arm64".to_string(), - other => other.to_ascii_lowercase(), - } -} - -#[derive(Debug, Eq, PartialEq)] -enum SupervisorBinSource { - Binary(PathBuf), - Image(String), -} - -fn resolve_supervisor_bin_source( - docker_config: &DockerComputeConfig, - current_exe: Option<&Path>, - target_candidates: &[PathBuf], -) -> CoreResult { - // Tier 1: explicit supervisor_bin in [openshell.drivers.docker]. - if let Some(path) = docker_config.supervisor_bin.clone() { - let path = canonicalize_existing_file(&path, "docker supervisor binary")?; - validate_linux_elf_binary(&path)?; - return Ok(SupervisorBinSource::Binary(path)); - } - - // Tier 2: explicit supervisor_image in [openshell.drivers.docker]. - // A configured image should be the source of truth even when a local - // developer build is present under target/. - if let Some(image) = docker_config.supervisor_image.clone() { - return Ok(SupervisorBinSource::Image(image)); - } - - // Tier 3: sibling `openshell-sandbox` next to the running gateway - // (release artifact layout). Linux-only because the sibling must be a - // Linux ELF to bind-mount into a Linux container. - if cfg!(target_os = "linux") - && let Some(current_exe) = current_exe - && let Some(parent) = current_exe.parent() - { - let sibling = parent.join("openshell-sandbox"); - if sibling.is_file() { - let path = canonicalize_existing_file(&sibling, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 4: local cargo target build (developer workflow). Preferred - // over the default registry image when available because it matches - // whatever the developer just built. - for candidate in target_candidates { - if candidate.is_file() { - let path = canonicalize_existing_file(candidate, "docker supervisor binary")?; - if validate_linux_elf_binary(&path).is_ok() { - return Ok(SupervisorBinSource::Binary(path)); - } - } - } - - // Tier 5: pull the release-matched default supervisor image and extract - // the binary to a host-side cache keyed by image content digest. - Ok(SupervisorBinSource::Image( - openshell_core::config::default_supervisor_image(), - )) -} - -pub(crate) async fn resolve_supervisor_bin( - docker: &Docker, - docker_config: &DockerComputeConfig, - daemon_arch: &str, -) -> CoreResult { - let current_exe = - if cfg!(target_os = "linux") - && docker_config.supervisor_bin.is_none() - && docker_config.supervisor_image.is_none() - { - Some(std::env::current_exe().map_err(|err| { - Error::config(format!("failed to resolve current executable: {err}")) - })?) - } else { - None - }; - let target_candidates = linux_supervisor_candidates(daemon_arch); - - match resolve_supervisor_bin_source(docker_config, current_exe.as_deref(), &target_candidates)? - { - SupervisorBinSource::Binary(path) => Ok(path), - SupervisorBinSource::Image(image) => { - extract_supervisor_bin_from_image(docker, &image).await - } - } -} - -fn linux_supervisor_candidates(daemon_arch: &str) -> Vec { - match daemon_arch { - "arm64" => vec![PathBuf::from( - "target/aarch64-unknown-linux-gnu/release/openshell-sandbox", - )], - "amd64" => vec![PathBuf::from( - "target/x86_64-unknown-linux-gnu/release/openshell-sandbox", - )], - _ => Vec::new(), - } -} - -/// Pull the supervisor image (if not already local), extract -/// `/openshell-sandbox` to a host cache keyed by the image's content -/// digest, and return the cache path. -/// -/// The extraction is atomic: the binary is written to a sibling temp file -/// inside the digest-keyed directory and renamed into place, so concurrent -/// gateway starts don't observe a partial file. -async fn extract_supervisor_bin_from_image(docker: &Docker, image: &str) -> CoreResult { - let refresh_attempted = if supervisor_image_should_refresh(image) { - info!(image = image, "Refreshing mutable docker supervisor image"); - match pull_supervisor_image(docker, image).await { - Ok(()) => true, - Err(err) => { - warn!( - image = image, - error = %err, - "failed to refresh mutable docker supervisor image; falling back to local image if present", - ); - true - } - } - } else { - false - }; - - // Inspect first to see if the image is already present; only pull on miss. - let inspect = match docker.inspect_image(image).await { - Ok(inspect) => inspect, - Err(err) if is_not_found_error(&err) && !refresh_attempted => { - info!(image = image, "Pulling docker supervisor image"); - pull_supervisor_image(docker, image).await?; - docker.inspect_image(image).await.map_err(|err| { - Error::config(format!( - "failed to inspect docker supervisor image '{image}' after pull: {err}", - )) - })? - } - Err(err) if is_not_found_error(&err) => { - return Err(Error::config(format!( - "docker supervisor image '{image}' is not present locally after refresh attempt", - ))); - } - Err(err) => { - return Err(Error::config(format!( - "failed to inspect docker supervisor image '{image}': {err}", - ))); - } - }; - - let digest = inspect.id.clone().ok_or_else(|| { - Error::config(format!( - "docker supervisor image '{image}' inspect response has no Id", - )) - })?; - - let cache_path = supervisor_cache_path(&digest)?; - if cache_path.is_file() { - validate_linux_elf_binary(&cache_path)?; - return Ok(cache_path); - } - - let cache_dir = cache_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - cache_path.display(), - )) - })?; - std::fs::create_dir_all(cache_dir).map_err(|err| { - Error::config(format!( - "failed to create docker supervisor cache dir '{}': {err}", - cache_dir.display(), - )) - })?; - - info!( - image = image, - digest = digest, - cache_path = %cache_path.display(), - "Extracting supervisor binary from image to host cache", - ); - - let binary_bytes = extract_supervisor_binary_bytes(docker, image).await?; - write_cache_binary_atomic(&cache_path, &binary_bytes)?; - validate_linux_elf_binary(&cache_path)?; - Ok(cache_path) -} - -async fn pull_supervisor_image(docker: &Docker, image: &str) -> CoreResult<()> { - let mut stream = docker.create_image( - Some(CreateImageOptions { - from_image: Some(image.to_string()), - ..Default::default() - }), - None, - None, - ); - while let Some(result) = stream.next().await { - result.map_err(|err| { - Error::config(format!( - "failed to pull docker supervisor image '{image}': {err}", - )) - })?; - } - Ok(()) -} - -/// Create a short-lived container from `image`, stream out the supervisor -/// binary as a tar archive, and return the untarred file bytes. The -/// container is always removed, even on error paths. -async fn extract_supervisor_binary_bytes(docker: &Docker, image: &str) -> CoreResult> { - let container_name = temp_extract_container_name(); - docker - .create_container( - Some( - CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(), - ), - ContainerCreateBody { - image: Some(image.to_string()), - entrypoint: Some(vec![SUPERVISOR_IMAGE_BINARY_PATH.to_string()]), - cmd: Some(Vec::new()), - ..Default::default() - }, - ) - .await - .map_err(|err| { - Error::config(format!( - "failed to create extractor container from '{image}': {err}", - )) - })?; - - // Always tear down the extractor container, even if extraction fails. - let result = download_binary_from_container(docker, &container_name).await; - if let Err(remove_err) = docker - .remove_container( - &container_name, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await - { - warn!( - container = container_name, - error = %remove_err, - "Failed to remove supervisor extractor container", - ); - } - result -} - -async fn download_binary_from_container( - docker: &Docker, - container_name: &str, -) -> CoreResult> { - let options = DownloadFromContainerOptionsBuilder::default() - .path(SUPERVISOR_IMAGE_BINARY_PATH) - .build(); - let mut stream = docker.download_from_container(container_name, Some(options)); - - let mut tar_bytes = Vec::new(); - while let Some(chunk) = stream.next().await { - let chunk: Bytes = chunk.map_err(|err| { - Error::config(format!( - "failed to read supervisor binary stream from '{container_name}': {err}", - )) - })?; - tar_bytes.extend_from_slice(&chunk); - } - - extract_first_tar_entry(&tar_bytes).map_err(|err| { - Error::config(format!( - "failed to extract supervisor binary from tar archive returned by '{container_name}': {err}", - )) - }) -} - -/// Extract the payload of the first regular-file entry in a tar archive. -/// Docker's `/containers//archive` endpoint returns a single-file tar -/// when `path` points to a file, so we only need the first entry. -fn extract_first_tar_entry(tar_bytes: &[u8]) -> Result, String> { - let mut archive = tar::Archive::new(std::io::Cursor::new(tar_bytes)); - let mut entries = archive - .entries() - .map_err(|err| format!("open tar archive: {err}"))?; - let mut entry = entries - .next() - .ok_or_else(|| "tar archive was empty".to_string())? - .map_err(|err| format!("read tar entry: {err}"))?; - let mut bytes = Vec::new(); - entry - .read_to_end(&mut bytes) - .map_err(|err| format!("read tar entry payload: {err}"))?; - Ok(bytes) -} - -fn write_cache_binary_atomic(final_path: &Path, bytes: &[u8]) -> CoreResult<()> { - let dir = final_path.parent().ok_or_else(|| { - Error::config(format!( - "docker supervisor cache path '{}' has no parent directory", - final_path.display(), - )) - })?; - let mut temp = tempfile::Builder::new() - .prefix(".openshell-sandbox-") - .tempfile_in(dir) - .map_err(|err| { - Error::config(format!( - "failed to create temp file for supervisor binary in '{}': {err}", - dir.display(), - )) - })?; - std::io::Write::write_all(&mut temp, bytes).map_err(|err| { - Error::config(format!( - "failed to write supervisor binary to temp file: {err}", - )) - })?; - temp.as_file().sync_all().map_err(|err| { - Error::config(format!("failed to sync supervisor binary temp file: {err}")) - })?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o755)).map_err( - |err| { - Error::config(format!( - "failed to chmod supervisor binary temp file: {err}", - )) - }, - )?; - } - - temp.persist(final_path).map_err(|err| { - Error::config(format!( - "failed to rename supervisor binary into '{}': {}", - final_path.display(), - err.error, - )) - })?; - Ok(()) -} - -/// Cache path for an extracted supervisor binary, keyed by the image's -/// content-addressable digest (e.g. `sha256:abc123…`). The digest-prefixed -/// directory keeps stale extractions from earlier releases isolated so they -/// can be GC'd without affecting the active binary. -fn supervisor_cache_path(digest: &str) -> CoreResult { - let base = openshell_core::paths::xdg_data_dir() - .map_err(|err| Error::config(format!("failed to resolve XDG data dir: {err}")))?; - Ok(supervisor_cache_path_with_base(&base, digest)) -} - -fn supervisor_cache_path_with_base(base: &Path, digest: &str) -> PathBuf { - let sanitized = digest.replace(':', "-"); - base.join("openshell") - .join("docker-supervisor") - .join(sanitized) - .join("openshell-sandbox") -} - -fn temp_extract_container_name() -> String { - use std::sync::atomic::{AtomicU64, Ordering}; - static SEQ: AtomicU64 = AtomicU64::new(0); - let pid = std::process::id(); - let seq = SEQ.fetch_add(1, Ordering::Relaxed); - format!("openshell-supervisor-extract-{pid}-{seq}") -} - -fn canonicalize_existing_file(path: &Path, description: &str) -> CoreResult { - if !path.is_file() { - return Err(Error::config(format!( - "{description} '{}' does not exist or is not a file", - path.display() - ))); - } - std::fs::canonicalize(path).map_err(|err| { - Error::config(format!( - "failed to resolve {description} '{}': {err}", - path.display() - )) - }) -} - -pub(crate) fn validate_linux_elf_binary(path: &Path) -> CoreResult<()> { - let mut file = std::fs::File::open(path).map_err(|err| { - Error::config(format!( - "failed to open docker supervisor binary '{}': {err}", - path.display() - )) - })?; - let mut magic = [0_u8; 4]; - file.read_exact(&mut magic).map_err(|err| { - Error::config(format!( - "failed to read docker supervisor binary '{}': {err}", - path.display() - )) - })?; - if magic != [0x7f, b'E', b'L', b'F'] { - return Err(Error::config(format!( - "docker supervisor binary '{}' must be a Linux ELF executable", - path.display() - ))); - } - Ok(()) -} - -fn docker_guest_tls_configured(docker_config: &DockerComputeConfig) -> bool { - docker_config.guest_tls_ca.is_some() - && docker_config.guest_tls_cert.is_some() - && docker_config.guest_tls_key.is_some() -} - -pub(crate) fn docker_guest_tls_paths( - docker_config: &DockerComputeConfig, -) -> CoreResult> { - let tls_flags_provided = docker_config.guest_tls_ca.is_some() - || docker_config.guest_tls_cert.is_some() - || docker_config.guest_tls_key.is_some(); - - if !docker_config.grpc_endpoint.starts_with("https://") { - if tls_flags_provided { - return Err(Error::config(format!( - "guest_tls_ca/guest_tls_cert/guest_tls_key were provided but grpc_endpoint is '{}'; TLS materials require an https:// endpoint", - docker_config.grpc_endpoint, - ))); - } - return Ok(None); - } - - let provided = [ - docker_config.guest_tls_ca.as_ref(), - docker_config.guest_tls_cert.as_ref(), - docker_config.guest_tls_key.as_ref(), - ]; - if provided.iter().all(Option::is_none) { - return Err(Error::config( - "docker compute driver requires guest_tls_ca, guest_tls_cert, and guest_tls_key when grpc_endpoint uses https://", - )); - } - - let Some(ca) = docker_config.guest_tls_ca.clone() else { - return Err(Error::config( - "guest_tls_ca is required when Docker sandbox TLS materials are configured", - )); - }; - let Some(cert) = docker_config.guest_tls_cert.clone() else { - return Err(Error::config( - "guest_tls_cert is required when Docker sandbox TLS materials are configured", - )); - }; - let Some(key) = docker_config.guest_tls_key.clone() else { - return Err(Error::config( - "guest_tls_key is required when Docker sandbox TLS materials are configured", - )); - }; - - Ok(Some(DockerGuestTlsPaths { - ca: canonicalize_existing_file(&ca, "docker TLS CA certificate")?, - cert: canonicalize_existing_file(&cert, "docker TLS client certificate")?, - key: canonicalize_existing_file(&key, "docker TLS client private key")?, - })) -} - -fn is_not_found_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 404, - .. - } - ) -} - -fn is_conflict_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 409, - .. - } - ) -} - -fn is_not_modified_error(err: &BollardError) -> bool { - matches!( - err, - BollardError::DockerResponseServerError { - status_code: 304, - .. - } - ) -} - -fn create_status_from_docker_error(operation: &str, err: BollardError) -> Status { - if matches!( - err, - BollardError::DockerResponseServerError { - status_code: 409, - .. - } - ) { - Status::already_exists("sandbox already exists") - } else { - internal_status(operation, err) - } -} - -fn internal_status(operation: &str, err: BollardError) -> Status { - Status::internal(format!("{operation} failed: {err}")) -} - -#[cfg(test)] -mod tests; diff --git a/crates/openshell-driver-docker/src/lib_win.rs b/crates/openshell-driver-docker/src/lib_win.rs new file mode 100644 index 000000000..343ce7c58 --- /dev/null +++ b/crates/openshell-driver-docker/src/lib_win.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod windows; + +pub use windows::{DockerComputeConfig, default_docker_supervisor_image}; diff --git a/crates/openshell-driver-kubernetes/Cargo.toml b/crates/openshell-driver-kubernetes/Cargo.toml index e72ac4f97..541f52619 100644 --- a/crates/openshell-driver-kubernetes/Cargo.toml +++ b/crates/openshell-driver-kubernetes/Cargo.toml @@ -3,6 +3,7 @@ [package] name = "openshell-driver-kubernetes" +autobins = false description = "Kubernetes compute driver for OpenShell" version.workspace = true edition.workspace = true @@ -12,7 +13,7 @@ repository.workspace = true [[bin]] name = "openshell-driver-kubernetes" -path = "src/main.rs" +path = "src/main_entry.rs" [dependencies] openshell-core = { path = "../openshell-core", default-features = false } diff --git a/crates/openshell-driver-kubernetes/src/main.rs b/crates/openshell-driver-kubernetes/src/main.rs index 5fffa026c..b7d5514ac 100644 --- a/crates/openshell-driver-kubernetes/src/main.rs +++ b/crates/openshell-driver-kubernetes/src/main.rs @@ -1,8 +1,11 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(not(target_os = "windows"))] -include!("main_unix.rs"); +use clap::{ArgAction, Parser}; +use miette::{IntoDiagnostic, Result}; +use std::net::SocketAddr; +use tracing::info; +use tracing_subscriber::EnvFilter; use openshell_core::VERSION; use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; diff --git a/crates/openshell-driver-kubernetes/src/main_entry.rs b/crates/openshell-driver-kubernetes/src/main_entry.rs new file mode 100644 index 000000000..432402121 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/main_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("main.rs"); + +#[cfg(target_os = "windows")] +include!("main_win.rs"); diff --git a/crates/openshell-driver-kubernetes/src/main_unix.rs b/crates/openshell-driver-kubernetes/src/main_unix.rs deleted file mode 100644 index c733b8a45..000000000 --- a/crates/openshell-driver-kubernetes/src/main_unix.rs +++ /dev/null @@ -1,181 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use clap::{ArgAction, Parser}; -use miette::{IntoDiagnostic, Result}; -use std::net::SocketAddr; -use tracing::info; -use tracing_subscriber::EnvFilter; - -use openshell_core::VERSION; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_kubernetes::{ - AppArmorProfile, ComputeDriverService, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, - KubernetesComputeConfig, KubernetesComputeDriver, KubernetesSidecarConfig, - SupervisorSideloadMethod, SupervisorTopology, -}; - -#[derive(Parser, Debug)] -#[command(name = "openshell-driver-kubernetes")] -#[command(version = VERSION)] -struct Args { - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_BIND", - default_value = "127.0.0.1:50061" - )] - bind_address: SocketAddr, - - #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] - log_level: String, - - #[arg(long, env = "OPENSHELL_SANDBOX_NAMESPACE", default_value = "default")] - sandbox_namespace: String, - - #[arg( - long, - env = "OPENSHELL_K8S_SANDBOX_SERVICE_ACCOUNT", - default_value = DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME - )] - sandbox_service_account: String, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] - sandbox_image: Option, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")] - sandbox_image_pull_policy: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_IMAGE_PULL_SECRETS", - value_delimiter = ',' - )] - sandbox_image_pull_secrets: Vec, - - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", - default_value = "/run/openshell/ssh.sock" - )] - sandbox_ssh_socket_path: String, - - #[arg(long, env = "OPENSHELL_CLIENT_TLS_SECRET_NAME")] - client_tls_secret_name: Option, - - #[arg(long, env = "OPENSHELL_HOST_GATEWAY_IP")] - host_gateway_ip: Option, - - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] - supervisor_image: Option, - - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")] - supervisor_image_pull_policy: Option, - - #[arg( - long, - env = "OPENSHELL_SUPERVISOR_SIDELOAD_METHOD", - default_value = "image-volume" - )] - supervisor_sideload_method: SupervisorSideloadMethod, - - #[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")] - topology: SupervisorTopology, - - #[arg( - long = "sidecar-proxy-uid", - alias = "proxy-uid", - env = "OPENSHELL_K8S_SIDECAR_PROXY_UID", - default_value_t = DEFAULT_PROXY_UID - )] - sidecar_proxy_uid: u32, - - #[arg( - long = "sidecar-process-binary-aware-network-policy", - env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY", - default_value_t = true, - action = ArgAction::Set - )] - sidecar_process_binary_aware_network_policy: bool, - - #[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")] - enable_user_namespaces: bool, - - #[arg(long, env = "OPENSHELL_K8S_APP_ARMOR_PROFILE")] - app_armor_profile: Option, - - /// Lifetime (seconds) of the projected `ServiceAccount` token - /// kubelet writes into each sandbox pod for the `IssueSandboxToken` - /// bootstrap exchange. Kubelet enforces a minimum of 600s; the - /// gateway clamps values outside `[600, 86400]`. Default 3600. - #[arg(long, env = "OPENSHELL_K8S_SA_TOKEN_TTL_SECS", default_value_t = 3600)] - sa_token_ttl_secs: i64, - - #[arg(long, env = "OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET")] - provider_spiffe_workload_api_socket_path: Option, - - #[arg(long, env = "OPENSHELL_K8S_SANDBOX_UID")] - sandbox_uid: Option, - - #[arg(long, env = "OPENSHELL_K8S_SANDBOX_GID")] - sandbox_gid: Option, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - tracing_subscriber::fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), - ) - .init(); - - let driver = KubernetesComputeDriver::new(KubernetesComputeConfig { - namespace: args.sandbox_namespace, - service_account_name: args.sandbox_service_account, - default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(), - image_pull_secrets: args.sandbox_image_pull_secrets, - supervisor_image: args - .supervisor_image - .unwrap_or_else(openshell_core::config::default_supervisor_image), - supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(), - supervisor_sideload_method: args.supervisor_sideload_method, - topology: args.topology, - sidecar: KubernetesSidecarConfig { - proxy_uid: args.sidecar_proxy_uid, - process_binary_aware_network_policy: args.sidecar_process_binary_aware_network_policy, - }, - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - ssh_socket_path: args.sandbox_ssh_socket_path, - client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(), - host_gateway_ip: args.host_gateway_ip.unwrap_or_default(), - enable_user_namespaces: args.enable_user_namespaces, - app_armor_profile: args.app_armor_profile, - workspace_default_storage_size: std::env::var( - "OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE", - ) - .unwrap_or_else(|_| { - openshell_driver_kubernetes::DEFAULT_WORKSPACE_STORAGE_SIZE.to_string() - }), - default_runtime_class_name: std::env::var("OPENSHELL_K8S_DEFAULT_RUNTIME_CLASS_NAME") - .unwrap_or_default(), - sa_token_ttl_secs: args.sa_token_ttl_secs, - provider_spiffe_workload_api_socket_path: args - .provider_spiffe_workload_api_socket_path - .unwrap_or_default(), - sandbox_uid: args.sandbox_uid, - sandbox_gid: args.sandbox_gid, - }) - .await - .into_diagnostic()?; - - info!(address = %args.bind_address, "Starting Kubernetes compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve(args.bind_address) - .await - .into_diagnostic() -} diff --git a/crates/openshell-driver-kubernetes/src/main_win.rs b/crates/openshell-driver-kubernetes/src/main_win.rs new file mode 100644 index 000000000..56cc95ac0 --- /dev/null +++ b/crates/openshell-driver-kubernetes/src/main_win.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + eprintln!("openshell-driver-kubernetes is unsupported on Windows"); + std::process::exit(1); +} diff --git a/crates/openshell-driver-podman/Cargo.toml b/crates/openshell-driver-podman/Cargo.toml index ef02c9a0f..e456c1d3c 100644 --- a/crates/openshell-driver-podman/Cargo.toml +++ b/crates/openshell-driver-podman/Cargo.toml @@ -3,6 +3,7 @@ [package] name = "openshell-driver-podman" +autobins = false description = "Podman compute driver for OpenShell" version.workspace = true edition.workspace = true @@ -10,9 +11,12 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +path = "src/lib_entry.rs" + [[bin]] name = "openshell-driver-podman" -path = "src/main.rs" +path = "src/main_entry.rs" [target.'cfg(target_os = "windows")'.dependencies] serde = { workspace = true } diff --git a/crates/openshell-driver-podman/src/lib.rs b/crates/openshell-driver-podman/src/lib.rs index b76c9e0fb..5847a10ea 100644 --- a/crates/openshell-driver-podman/src/lib.rs +++ b/crates/openshell-driver-podman/src/lib.rs @@ -1,11 +1,15 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -mod windows; +pub(crate) mod client; +pub mod config; +pub(crate) mod container; +pub mod driver; +pub mod grpc; +#[cfg(test)] +pub(crate) mod test_utils; +pub(crate) mod watcher; -#[cfg(target_os = "windows")] -pub use windows::PodmanComputeConfig; - -#[cfg(not(target_os = "windows"))] -include!("lib_unix.rs"); +pub use config::PodmanComputeConfig; +pub use driver::PodmanComputeDriver; +pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-podman/src/lib_entry.rs b/crates/openshell-driver-podman/src/lib_entry.rs new file mode 100644 index 000000000..4272e7c53 --- /dev/null +++ b/crates/openshell-driver-podman/src/lib_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("lib.rs"); + +#[cfg(target_os = "windows")] +include!("lib_win.rs"); diff --git a/crates/openshell-driver-podman/src/lib_unix.rs b/crates/openshell-driver-podman/src/lib_unix.rs deleted file mode 100644 index 5847a10ea..000000000 --- a/crates/openshell-driver-podman/src/lib_unix.rs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -pub(crate) mod client; -pub mod config; -pub(crate) mod container; -pub mod driver; -pub mod grpc; -#[cfg(test)] -pub(crate) mod test_utils; -pub(crate) mod watcher; - -pub use config::PodmanComputeConfig; -pub use driver::PodmanComputeDriver; -pub use grpc::ComputeDriverService; diff --git a/crates/openshell-driver-podman/src/lib_win.rs b/crates/openshell-driver-podman/src/lib_win.rs new file mode 100644 index 000000000..514eed1bd --- /dev/null +++ b/crates/openshell-driver-podman/src/lib_win.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod windows; + +pub use windows::PodmanComputeConfig; diff --git a/crates/openshell-driver-podman/src/main.rs b/crates/openshell-driver-podman/src/main.rs index 42060fd75..e28707588 100644 --- a/crates/openshell-driver-podman/src/main.rs +++ b/crates/openshell-driver-podman/src/main.rs @@ -1,11 +1,185 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -fn main() { - eprintln!("openshell-driver-podman is unsupported on Windows"); - std::process::exit(1); +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use std::net::SocketAddr; +use std::path::PathBuf; +use tracing::info; +use tracing_subscriber::EnvFilter; + +use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_podman::config::{ + DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, + ImagePullPolicy, +}; +use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; + +#[derive(Parser)] +#[command(name = "openshell-driver-podman")] +#[command(version = VERSION)] +struct Args { + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_BIND", + default_value = "127.0.0.1:50061" + )] + bind_address: SocketAddr, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + /// Path to the Podman API Unix socket. + #[arg(long, env = "OPENSHELL_PODMAN_SOCKET")] + podman_socket: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] + sandbox_image: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", + default_value_t = ImagePullPolicy::Missing + )] + sandbox_image_pull_policy: ImagePullPolicy, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + grpc_endpoint: Option, + + /// Port the gateway server is listening on. + /// + /// Used when `--grpc-endpoint` is not set to auto-detect the endpoint + /// that sandbox containers dial back to. + #[arg( + long, + env = "OPENSHELL_GATEWAY_PORT", + default_value_t = openshell_core::config::DEFAULT_SERVER_PORT + )] + gateway_port: u16, + + /// Host gateway IP used for sandbox host aliases. + /// + /// Empty uses Podman's `host-gateway` resolver. + #[arg(long, env = "OPENSHELL_PODMAN_HOST_GATEWAY_IP")] + host_gateway_ip: Option, + + #[arg( + long, + env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", + default_value = openshell_core::container_paths::SSH_SOCKET_PATH + )] + sandbox_ssh_socket_path: String, + + /// Podman bridge network name. + #[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)] + network_name: String, + + /// Container stop timeout in seconds (SIGTERM → SIGKILL). + #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] + stop_timeout: u32, + + /// Container cgroup PID limit for sandbox containers. Set 0 to inherit + /// Podman's runtime/default PID limit. + #[arg( + long, + env = "OPENSHELL_SANDBOX_PIDS_LIMIT", + default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT + )] + sandbox_pids_limit: i64, + + /// OCI image containing the openshell-sandbox supervisor binary. + #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] + supervisor_image: Option, + + /// Host path to the CA certificate for sandbox mTLS. + #[arg(long, env = "OPENSHELL_PODMAN_TLS_CA")] + podman_tls_ca: Option, + + /// Host path to the client certificate for sandbox mTLS. + #[arg(long, env = "OPENSHELL_PODMAN_TLS_CERT")] + podman_tls_cert: Option, + + /// Host path to the client private key for sandbox mTLS. + #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] + podman_tls_key: Option, + + /// Corporate forward proxy URL for the supervisor's upstream TLS dials, + /// in explicit `http://host:port` form (scheme and port required). + /// Credentials must not be embedded in the URL; use + /// `--sandbox-proxy-auth-file` instead. + #[arg(long, env = "OPENSHELL_SANDBOX_HTTPS_PROXY")] + sandbox_https_proxy: Option, + + /// Comma-separated `NO_PROXY` list injected alongside the proxy URL. + #[arg(long, env = "OPENSHELL_SANDBOX_NO_PROXY")] + sandbox_no_proxy: Option, + + /// Path to a file containing the corporate proxy credentials as + /// `user:pass`. Delivered to the supervisor through a root-only secret + /// mount so the credentials never appear in config or container metadata. + #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_FILE")] + sandbox_proxy_auth_file: Option, + + /// Explicit acknowledgement (`true`) that the proxy credential is sent + /// as cleartext Basic auth over the plain-TCP connection to the http:// + /// proxy. Required when `--sandbox-proxy-auth-file` is set. + #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE")] + sandbox_proxy_auth_allow_insecure: Option, + + /// Send the destination hostname in CONNECT requests to the corporate + /// proxy instead of a validated IP. Only for proxies whose ACLs filter + /// on hostnames: the proxy then resolves the name itself, so sandbox + /// SSRF/`allowed_ips` validation no longer binds the connection. + #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")] + sandbox_proxy_connect_by_hostname: Option, } -#[cfg(not(target_os = "windows"))] -include!("main_unix.rs"); +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + tracing_subscriber::fmt() + .with_env_filter( + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), + ) + .init(); + + let driver = PodmanComputeDriver::new(PodmanComputeConfig { + socket_path: args.podman_socket, + default_image: args.sandbox_image.unwrap_or_default(), + image_pull_policy: args.sandbox_image_pull_policy, + grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), + gateway_port: args.gateway_port, + host_gateway_ip: args + .host_gateway_ip + .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), + sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, + network_name: args.network_name, + stop_timeout_secs: args.stop_timeout, + supervisor_image: args + .supervisor_image + .unwrap_or_else(openshell_core::config::default_supervisor_image), + guest_tls_ca: args.podman_tls_ca, + guest_tls_cert: args.podman_tls_cert, + guest_tls_key: args.podman_tls_key, + sandbox_pids_limit: args.sandbox_pids_limit, + https_proxy: args.sandbox_https_proxy, + no_proxy: args.sandbox_no_proxy, + proxy_auth_file: args.sandbox_proxy_auth_file, + proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure, + proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname, + ..PodmanComputeConfig::default() + }) + .await + .into_diagnostic()?; + + info!(address = %args.bind_address, "Starting Podman compute driver"); + tonic::transport::Server::builder() + .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) + .serve_with_shutdown(args.bind_address, async { + tokio::signal::ctrl_c().await.ok(); + info!("Received shutdown signal, draining in-flight requests"); + }) + .await + .into_diagnostic() +} diff --git a/crates/openshell-driver-podman/src/main_entry.rs b/crates/openshell-driver-podman/src/main_entry.rs new file mode 100644 index 000000000..432402121 --- /dev/null +++ b/crates/openshell-driver-podman/src/main_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("main.rs"); + +#[cfg(target_os = "windows")] +include!("main_win.rs"); diff --git a/crates/openshell-driver-podman/src/main_unix.rs b/crates/openshell-driver-podman/src/main_unix.rs deleted file mode 100644 index e28707588..000000000 --- a/crates/openshell-driver-podman/src/main_unix.rs +++ /dev/null @@ -1,185 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use clap::Parser; -use miette::{IntoDiagnostic, Result}; -use std::net::SocketAddr; -use std::path::PathBuf; -use tracing::info; -use tracing_subscriber::EnvFilter; - -use openshell_core::VERSION; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_podman::config::{ - DEFAULT_NETWORK_NAME, DEFAULT_PODMAN_STOP_TIMEOUT_SECS, DEFAULT_SANDBOX_PIDS_LIMIT, - ImagePullPolicy, -}; -use openshell_driver_podman::{ComputeDriverService, PodmanComputeConfig, PodmanComputeDriver}; - -#[derive(Parser)] -#[command(name = "openshell-driver-podman")] -#[command(version = VERSION)] -struct Args { - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_BIND", - default_value = "127.0.0.1:50061" - )] - bind_address: SocketAddr, - - #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] - log_level: String, - - /// Path to the Podman API Unix socket. - #[arg(long, env = "OPENSHELL_PODMAN_SOCKET")] - podman_socket: Option, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE")] - sandbox_image: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY", - default_value_t = ImagePullPolicy::Missing - )] - sandbox_image_pull_policy: ImagePullPolicy, - - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - grpc_endpoint: Option, - - /// Port the gateway server is listening on. - /// - /// Used when `--grpc-endpoint` is not set to auto-detect the endpoint - /// that sandbox containers dial back to. - #[arg( - long, - env = "OPENSHELL_GATEWAY_PORT", - default_value_t = openshell_core::config::DEFAULT_SERVER_PORT - )] - gateway_port: u16, - - /// Host gateway IP used for sandbox host aliases. - /// - /// Empty uses Podman's `host-gateway` resolver. - #[arg(long, env = "OPENSHELL_PODMAN_HOST_GATEWAY_IP")] - host_gateway_ip: Option, - - #[arg( - long, - env = "OPENSHELL_SANDBOX_SSH_SOCKET_PATH", - default_value = openshell_core::container_paths::SSH_SOCKET_PATH - )] - sandbox_ssh_socket_path: String, - - /// Podman bridge network name. - #[arg(long, env = "OPENSHELL_NETWORK_NAME", default_value = DEFAULT_NETWORK_NAME)] - network_name: String, - - /// Container stop timeout in seconds (SIGTERM → SIGKILL). - #[arg(long, env = "OPENSHELL_STOP_TIMEOUT", default_value_t = DEFAULT_PODMAN_STOP_TIMEOUT_SECS)] - stop_timeout: u32, - - /// Container cgroup PID limit for sandbox containers. Set 0 to inherit - /// Podman's runtime/default PID limit. - #[arg( - long, - env = "OPENSHELL_SANDBOX_PIDS_LIMIT", - default_value_t = DEFAULT_SANDBOX_PIDS_LIMIT - )] - sandbox_pids_limit: i64, - - /// OCI image containing the openshell-sandbox supervisor binary. - #[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE")] - supervisor_image: Option, - - /// Host path to the CA certificate for sandbox mTLS. - #[arg(long, env = "OPENSHELL_PODMAN_TLS_CA")] - podman_tls_ca: Option, - - /// Host path to the client certificate for sandbox mTLS. - #[arg(long, env = "OPENSHELL_PODMAN_TLS_CERT")] - podman_tls_cert: Option, - - /// Host path to the client private key for sandbox mTLS. - #[arg(long, env = "OPENSHELL_PODMAN_TLS_KEY")] - podman_tls_key: Option, - - /// Corporate forward proxy URL for the supervisor's upstream TLS dials, - /// in explicit `http://host:port` form (scheme and port required). - /// Credentials must not be embedded in the URL; use - /// `--sandbox-proxy-auth-file` instead. - #[arg(long, env = "OPENSHELL_SANDBOX_HTTPS_PROXY")] - sandbox_https_proxy: Option, - - /// Comma-separated `NO_PROXY` list injected alongside the proxy URL. - #[arg(long, env = "OPENSHELL_SANDBOX_NO_PROXY")] - sandbox_no_proxy: Option, - - /// Path to a file containing the corporate proxy credentials as - /// `user:pass`. Delivered to the supervisor through a root-only secret - /// mount so the credentials never appear in config or container metadata. - #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_FILE")] - sandbox_proxy_auth_file: Option, - - /// Explicit acknowledgement (`true`) that the proxy credential is sent - /// as cleartext Basic auth over the plain-TCP connection to the http:// - /// proxy. Required when `--sandbox-proxy-auth-file` is set. - #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE")] - sandbox_proxy_auth_allow_insecure: Option, - - /// Send the destination hostname in CONNECT requests to the corporate - /// proxy instead of a validated IP. Only for proxies whose ACLs filter - /// on hostnames: the proxy then resolves the name itself, so sandbox - /// SSRF/`allowed_ips` validation no longer binds the connection. - #[arg(long, env = "OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME")] - sandbox_proxy_connect_by_hostname: Option, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - tracing_subscriber::fmt() - .with_env_filter( - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)), - ) - .init(); - - let driver = PodmanComputeDriver::new(PodmanComputeConfig { - socket_path: args.podman_socket, - default_image: args.sandbox_image.unwrap_or_default(), - image_pull_policy: args.sandbox_image_pull_policy, - grpc_endpoint: args.grpc_endpoint.unwrap_or_default(), - gateway_port: args.gateway_port, - host_gateway_ip: args - .host_gateway_ip - .unwrap_or_else(PodmanComputeConfig::default_host_gateway_ip), - sandbox_ssh_socket_path: args.sandbox_ssh_socket_path, - network_name: args.network_name, - stop_timeout_secs: args.stop_timeout, - supervisor_image: args - .supervisor_image - .unwrap_or_else(openshell_core::config::default_supervisor_image), - guest_tls_ca: args.podman_tls_ca, - guest_tls_cert: args.podman_tls_cert, - guest_tls_key: args.podman_tls_key, - sandbox_pids_limit: args.sandbox_pids_limit, - https_proxy: args.sandbox_https_proxy, - no_proxy: args.sandbox_no_proxy, - proxy_auth_file: args.sandbox_proxy_auth_file, - proxy_auth_allow_insecure: args.sandbox_proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.sandbox_proxy_connect_by_hostname, - ..PodmanComputeConfig::default() - }) - .await - .into_diagnostic()?; - - info!(address = %args.bind_address, "Starting Podman compute driver"); - tonic::transport::Server::builder() - .add_service(ComputeDriverServer::new(ComputeDriverService::new(driver))) - .serve_with_shutdown(args.bind_address, async { - tokio::signal::ctrl_c().await.ok(); - info!("Received shutdown signal, draining in-flight requests"); - }) - .await - .into_diagnostic() -} diff --git a/crates/openshell-driver-podman/src/main_win.rs b/crates/openshell-driver-podman/src/main_win.rs new file mode 100644 index 000000000..527dacccb --- /dev/null +++ b/crates/openshell-driver-podman/src/main_win.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + eprintln!("openshell-driver-podman is unsupported on Windows"); + std::process::exit(1); +} diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index 3a00edc44..eee1dccda 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -3,6 +3,7 @@ [package] name = "openshell-driver-vm" +autobins = false description = "MicroVM compute driver for OpenShell" version.workspace = true edition.workspace = true @@ -12,11 +13,11 @@ repository.workspace = true [lib] name = "openshell_driver_vm" -path = "src/lib.rs" +path = "src/lib_entry.rs" [[bin]] name = "openshell-driver-vm" -path = "src/main.rs" +path = "src/main_entry.rs" [target.'cfg(not(target_os = "windows"))'.dependencies] openshell-core = { path = "../openshell-core", default-features = false } diff --git a/crates/openshell-driver-vm/src/lib.rs b/crates/openshell-driver-vm/src/lib.rs index e0d663bc4..98ba6b0c9 100644 --- a/crates/openshell-driver-vm/src/lib.rs +++ b/crates/openshell-driver-vm/src/lib.rs @@ -1,10 +1,24 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -pub fn windows_unsupported() -> &'static str { - "openshell-driver-vm is unsupported on Windows" -} +pub mod driver; +mod embedded_runtime; +mod ffi; +pub mod gpu; +pub mod lifecycle; +mod nft_ruleset; +pub mod otel_tracing; +pub mod procguard; +mod rootfs; +mod runtime; -#[cfg(not(target_os = "windows"))] -include!("lib_unix.rs"); +pub use driver::{VmDriver, VmDriverConfig}; +pub use lifecycle::{ + BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason, + LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult, + RestoreContext, +}; +pub use runtime::{ + VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, + configured_runtime_dir, run_vm, +}; diff --git a/crates/openshell-driver-vm/src/lib_entry.rs b/crates/openshell-driver-vm/src/lib_entry.rs new file mode 100644 index 000000000..4272e7c53 --- /dev/null +++ b/crates/openshell-driver-vm/src/lib_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("lib.rs"); + +#[cfg(target_os = "windows")] +include!("lib_win.rs"); diff --git a/crates/openshell-driver-vm/src/lib_unix.rs b/crates/openshell-driver-vm/src/lib_unix.rs deleted file mode 100644 index 98ba6b0c9..000000000 --- a/crates/openshell-driver-vm/src/lib_unix.rs +++ /dev/null @@ -1,24 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -pub mod driver; -mod embedded_runtime; -mod ffi; -pub mod gpu; -pub mod lifecycle; -mod nft_ruleset; -pub mod otel_tracing; -pub mod procguard; -mod rootfs; -mod runtime; - -pub use driver::{VmDriver, VmDriverConfig}; -pub use lifecycle::{ - BackendFeature, ExtensionCapabilities, ExtensionDescriptor, GuestInitDropin, LaunchAbortReason, - LaunchPlan, LifecycleError, LifecycleExtension, LifecycleExtensionRegistry, LifecycleResult, - RestoreContext, -}; -pub use runtime::{ - VM_RUNTIME_DIR_ENV, VmBackend, VmLaunchConfig, cleanup_stale_tap_interfaces, - configured_runtime_dir, run_vm, -}; diff --git a/crates/openshell-driver-vm/src/lib_win.rs b/crates/openshell-driver-vm/src/lib_win.rs new file mode 100644 index 000000000..55f725cee --- /dev/null +++ b/crates/openshell-driver-vm/src/lib_win.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub fn windows_unsupported() -> &'static str { + "openshell-driver-vm is unsupported on Windows" +} diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index c41b94f24..949d4ce05 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -1,11 +1,782 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -fn main() { - eprintln!("openshell-driver-vm is unsupported on Windows"); - std::process::exit(1); +use clap::Parser; +use futures::Stream; +use miette::{IntoDiagnostic, Result}; +use openshell_core::VERSION; +use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; +use openshell_driver_vm::otel_tracing::compute_driver_rpc_layer; +#[cfg(target_os = "macos")] +use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; +use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; +use std::io; +use std::net::SocketAddr; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::task::{Context, Poll}; +use tokio::net::{UnixListener, UnixStream}; +use tracing::info; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::prelude::*; + +#[derive(Parser, Debug)] +#[command(name = "openshell-driver-vm")] +#[command(version = VERSION)] +#[allow(clippy::struct_excessive_bools)] +struct Args { + #[arg(long, hide = true, default_value_t = false)] + internal_run_vm: bool, + + #[arg(long = "vm-root-disk", hide = true, alias = "vm-rootfs")] + vm_root_disk: Option, + + #[arg(long = "vm-overlay-disk", hide = true)] + vm_overlay_disk: Option, + + #[arg(long = "vm-image-disk", hide = true)] + vm_image_disk: Option, + + #[arg(long = "vm-kernel-image", hide = true)] + vm_kernel_image: Option, + + #[arg(long, hide = true)] + vm_exec: Option, + + #[arg(long, hide = true, default_value = "/")] + vm_workdir: String, + + #[arg(long, hide = true)] + vm_env: Vec, + + #[arg(long, hide = true)] + vm_console_output: Option, + + #[arg(long, hide = true, default_value_t = 2)] + vm_vcpus: u8, + + #[arg(long, hide = true, default_value_t = 2048)] + vm_mem_mib: u32, + + #[arg(long, hide = true, default_value_t = 1)] + vm_krun_log_level: u32, + + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_BIND")] + bind_address: Option, + + #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] + bind_socket: Option, + + #[arg(long, hide = true)] + expected_peer_pid: Option, + + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_UNAUTHENTICATED_TCP", + default_value_t = false + )] + allow_unauthenticated_tcp: bool, + + #[arg( + long, + env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_SAME_UID_PEER", + default_value_t = false + )] + allow_same_uid_peer: bool, + + #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] + log_level: String, + + #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] + otlp_endpoint: Option, + + #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] + openshell_endpoint: Option, + + #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] + default_image: String, + + #[arg(long, env = "OPENSHELL_VM_BOOTSTRAP_IMAGE", default_value = "")] + bootstrap_image: String, + + #[arg( + long, + env = "OPENSHELL_VM_DRIVER_STATE_DIR", + default_value = "target/openshell-vm-driver" + )] + state_dir: PathBuf, + + #[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")] + guest_tls_ca: Option, + + #[arg(long = "guest-tls-cert", env = "OPENSHELL_VM_TLS_CERT")] + guest_tls_cert: Option, + + #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] + guest_tls_key: Option, + + #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] + krun_log_level: u32, + + #[arg(long, env = "OPENSHELL_VM_DRIVER_VCPUS", default_value_t = 2)] + vcpus: u8, + + #[arg(long, env = "OPENSHELL_VM_DRIVER_MEM_MIB", default_value_t = 2048)] + mem_mib: u32, + + #[arg(long, env = "OPENSHELL_VM_OVERLAY_DISK_MIB", default_value_t = 4096)] + overlay_disk_mib: u64, + + #[arg(long, env = "OPENSHELL_VM_GPU")] + gpu: bool, + + #[arg(long, env = "OPENSHELL_VM_GPU_MEM_MIB", default_value_t = 8192)] + gpu_mem_mib: u32, + + #[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)] + gpu_vcpus: u8, + + #[arg(long, env = "OPENSHELL_VM_SANDBOX_UID")] + sandbox_uid: Option, + + #[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")] + sandbox_gid: Option, + + #[arg(long, hide = true)] + vm_backend: Option, + + #[arg(long, hide = true)] + vm_gpu_bdf: Option, + + #[arg(long, hide = true)] + vm_tap_device: Option, + + #[arg(long, hide = true)] + vm_guest_ip: Option, + + #[arg(long, hide = true)] + vm_host_ip: Option, + + #[arg(long, hide = true)] + vm_vsock_cid: Option, + + #[arg(long, hide = true)] + vm_guest_mac: Option, + + #[arg(long, hide = true)] + vm_gateway_port: Option, } -#[cfg(not(target_os = "windows"))] -include!("main_unix.rs"); +#[tokio::main] +async fn main() -> Result<()> { + let args = Args::parse(); + if args.internal_run_vm { + // We intentionally defer procguard arming until `run_vm()` so + // that the only arm is the one that knows how to clean up + // gvproxy. Racing two watchers against the same parent-death + // event causes the bare arm's `exit(1)` to win, skipping the + // gvproxy cleanup and leaking the helper. The risk window + // before `run_vm` arms procguard is ~a few syscalls long + // (`build_vm_launch_config`, `configured_runtime_dir`), which + // is negligible next to the parent gRPC server's uptime. + maybe_reexec_internal_vm_with_runtime_env()?; + let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; + run_vm(&config).map_err(|err| miette::miette!("{err}"))?; + return Ok(()); + } + + let (tracer_provider, setup_error) = + openshell_driver_vm::otel_tracing::provider_for(args.otlp_endpoint.as_deref()); + tracing_subscriber::registry() + .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level))) + .with(tracing_subscriber::fmt::layer()) + .with( + tracer_provider + .as_ref() + .map(openshell_driver_vm::otel_tracing::layer), + ) + .init(); + if let Some(error) = setup_error { + tracing::error!(%error, "OTLP exporting could not be started"); + } else if let Some(endpoint) = &args.otlp_endpoint { + info!(endpoint, "OTLP exporting enabled"); + } + + let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?; + + // Arm procguard so that if the gateway is killed (SIGKILL or crash) + // we also die. Without this the driver is reparented to init and + // keeps its per-sandbox VM launchers alive forever. Launchers have + // their own procguards (armed in `run_vm`) which cascade cleanup of + // gvproxy and the libkrun worker the moment this driver exits. + if let Err(err) = procguard::die_with_parent() { + tracing::warn!( + error = %err, + "procguard arm failed; gateway crashes may orphan this driver" + ); + } + + let driver = VmDriver::new(VmDriverConfig { + openshell_endpoint: args + .openshell_endpoint + .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, + state_dir: args.state_dir.clone(), + launcher_bin: None, + default_image: args.default_image.clone(), + bootstrap_image: args.bootstrap_image.clone(), + log_level: args.log_level.clone(), + krun_log_level: args.krun_log_level, + vcpus: args.vcpus, + mem_mib: args.mem_mib, + overlay_disk_mib: args.overlay_disk_mib, + guest_tls_ca: args.guest_tls_ca.clone(), + guest_tls_cert: args.guest_tls_cert.clone(), + guest_tls_key: args.guest_tls_key.clone(), + gpu_enabled: args.gpu, + gpu_mem_mib: args.gpu_mem_mib, + gpu_vcpus: args.gpu_vcpus, + sandbox_uid: args.sandbox_uid, + sandbox_gid: args.sandbox_gid, + }) + .await + .map_err(|err| miette::miette!("{err}"))?; + + let result = match listen_mode { + ComputeDriverListenMode::Unix { + socket_path, + expected_peer_pid, + } => { + prepare_compute_driver_socket(&socket_path).map_err(|err| miette::miette!("{err}"))?; + + info!(socket = %socket_path.display(), "Starting vm compute driver"); + let listener = UnixListener::bind(&socket_path).into_diagnostic()?; + restrict_socket_permissions(&socket_path).map_err(|err| miette::miette!("{err}"))?; + let result = tonic::transport::Server::builder() + .layer(compute_driver_rpc_layer()) + .add_service(ComputeDriverServer::new(driver)) + .serve_with_incoming_shutdown( + AuthenticatedUnixIncoming::new(listener, expected_peer_pid), + shutdown_signal(), + ) + .await + .into_diagnostic(); + let _ = std::fs::remove_file(&socket_path); + result + } + ComputeDriverListenMode::Tcp(bind_address) => { + info!(address = %bind_address, "Starting unauthenticated dev vm compute driver"); + tonic::transport::Server::builder() + .layer(compute_driver_rpc_layer()) + .add_service(ComputeDriverServer::new(driver)) + .serve_with_shutdown(bind_address, shutdown_signal()) + .await + .into_diagnostic() + } + }; + if let Some(provider) = &tracer_provider + && let Err(error) = provider.shutdown() + { + tracing::warn!(%error, "OTLP tracer provider shutdown failed"); + } + result +} + +async fn shutdown_signal() { + let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("install SIGTERM handler"); + tokio::select! { + result = tokio::signal::ctrl_c() => { + if let Err(error) = result { + tracing::warn!(%error, "failed to listen for Ctrl-C"); + } + } + _ = terminate.recv() => {} + } + info!("Shutdown signal received; stopping vm compute driver"); +} + +#[derive(Debug, Clone, PartialEq, Eq)] +enum ComputeDriverListenMode { + Unix { + socket_path: PathBuf, + expected_peer_pid: Option, + }, + Tcp(SocketAddr), +} + +fn compute_driver_listen_mode(args: &Args) -> std::result::Result { + if let Some(socket_path) = args.bind_socket.clone() { + if args.expected_peer_pid.is_none() && !args.allow_same_uid_peer { + return Err( + "--expected-peer-pid is required with --bind-socket; use --allow-same-uid-peer only for local development" + .to_string(), + ); + } + return Ok(ComputeDriverListenMode::Unix { + socket_path, + expected_peer_pid: args.expected_peer_pid, + }); + } + + if !args.allow_unauthenticated_tcp { + return Err( + "--bind-socket is required; unauthenticated TCP mode is disabled unless --allow-unauthenticated-tcp is set for local development" + .to_string(), + ); + } + + let Some(bind_address) = args.bind_address else { + return Err("--bind-address is required with --allow-unauthenticated-tcp".to_string()); + }; + + Ok(ComputeDriverListenMode::Tcp(bind_address)) +} + +fn prepare_compute_driver_socket(socket_path: &Path) -> std::result::Result<(), String> { + let Some(parent) = socket_path.parent() else { + return Err(format!( + "vm compute driver socket path '{}' has no parent directory", + socket_path.display() + )); + }; + let expected_uid = current_euid(); + prepare_private_socket_dir(parent, expected_uid)?; + remove_stale_socket(socket_path, expected_uid) +} + +fn current_euid() -> u32 { + rustix::process::geteuid().as_raw() +} + +fn prepare_private_socket_dir( + socket_dir: &Path, + expected_uid: u32, +) -> std::result::Result<(), String> { + std::fs::create_dir_all(socket_dir) + .map_err(|err| format!("create socket dir {}: {err}", socket_dir.display()))?; + let metadata = std::fs::symlink_metadata(socket_dir) + .map_err(|err| format!("stat socket dir {}: {err}", socket_dir.display()))?; + let file_type = metadata.file_type(); + if file_type.is_symlink() { + return Err(format!( + "socket dir {} is a symlink; refusing to use it", + socket_dir.display() + )); + } + if !file_type.is_dir() { + return Err(format!( + "socket dir {} is not a directory", + socket_dir.display() + )); + } + if metadata.uid() != expected_uid { + return Err(format!( + "socket dir {} is owned by uid {} but current euid is {}", + socket_dir.display(), + metadata.uid(), + expected_uid + )); + } + std::fs::set_permissions(socket_dir, std::fs::Permissions::from_mode(0o700)) + .map_err(|err| format!("chmod socket dir {}: {err}", socket_dir.display())) +} + +fn remove_stale_socket(socket_path: &Path, expected_uid: u32) -> std::result::Result<(), String> { + let metadata = match std::fs::symlink_metadata(socket_path) { + Ok(metadata) => metadata, + Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(err) => return Err(format!("stat socket {}: {err}", socket_path.display())), + }; + let file_type = metadata.file_type(); + if file_type.is_symlink() { + return Err(format!( + "socket {} is a symlink; refusing to remove it", + socket_path.display() + )); + } + if metadata.uid() != expected_uid { + return Err(format!( + "socket {} is owned by uid {} but current euid is {}", + socket_path.display(), + metadata.uid(), + expected_uid + )); + } + if !file_type.is_socket() { + return Err(format!( + "socket path {} exists but is not a Unix socket", + socket_path.display() + )); + } + std::fs::remove_file(socket_path) + .map_err(|err| format!("remove stale socket {}: {err}", socket_path.display())) +} + +fn restrict_socket_permissions(socket_path: &Path) -> std::result::Result<(), String> { + std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600)) + .map_err(|err| format!("chmod socket {}: {err}", socket_path.display())) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct PeerCredentials { + uid: u32, + pid: Option, +} + +fn peer_credentials(stream: &UnixStream) -> std::result::Result { + let credentials = stream + .peer_cred() + .map_err(|err| format!("read peer credentials: {err}"))?; + Ok(PeerCredentials { + uid: credentials.uid(), + pid: credentials.pid(), + }) +} + +fn authorize_peer_credentials( + peer: PeerCredentials, + driver_uid: u32, + gateway_pid: Option, +) -> std::result::Result<(), String> { + if peer.uid != driver_uid { + return Err(format!( + "peer uid {} does not match current euid {}", + peer.uid, driver_uid + )); + } + let Some(gateway_pid) = gateway_pid else { + return Ok(()); + }; + let Some(peer_process_id) = peer.pid.and_then(|pid| u32::try_from(pid).ok()) else { + return Err(format!( + "peer pid is unavailable; expected gateway pid {gateway_pid}" + )); + }; + if peer_process_id != gateway_pid { + return Err(format!( + "peer pid {peer_process_id} does not match expected gateway pid {gateway_pid}" + )); + } + Ok(()) +} + +struct AuthenticatedUnixIncoming { + listener: UnixListener, + expected_uid: u32, + expected_peer_pid: Option, +} + +impl AuthenticatedUnixIncoming { + fn new(listener: UnixListener, expected_peer_pid: Option) -> Self { + Self { + listener, + expected_uid: current_euid(), + expected_peer_pid, + } + } +} + +impl Stream for AuthenticatedUnixIncoming { + type Item = io::Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + match this.listener.poll_accept(cx) { + Poll::Ready(Ok((stream, _addr))) => { + let authorized = peer_credentials(&stream).and_then(|peer| { + authorize_peer_credentials(peer, this.expected_uid, this.expected_peer_pid) + }); + match authorized { + Ok(()) => return Poll::Ready(Some(Ok(stream))), + Err(err) => { + tracing::warn!( + error = %err, + "rejected vm compute driver UDS client" + ); + } + } + } + Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), + Poll::Pending => return Poll::Pending, + } + } + } +} + +fn build_vm_launch_config(args: &Args) -> std::result::Result { + let root_disk = args + .vm_root_disk + .clone() + .ok_or_else(|| "--vm-root-disk is required in internal VM mode".to_string())?; + let overlay_disk = args + .vm_overlay_disk + .clone() + .ok_or_else(|| "--vm-overlay-disk is required in internal VM mode".to_string())?; + let image_disk = args.vm_image_disk.clone(); + let exec_path = args + .vm_exec + .clone() + .ok_or_else(|| "--vm-exec is required in internal VM mode".to_string())?; + let console_output = args + .vm_console_output + .clone() + .ok_or_else(|| "--vm-console-output is required in internal VM mode".to_string())?; + + let backend = match args.vm_backend.as_deref() { + Some("qemu") => VmBackend::Qemu, + Some("libkrun") | None => VmBackend::Libkrun, + Some(other) => return Err(format!("unknown VM backend: {other}")), + }; + + Ok(VmLaunchConfig { + root_disk, + overlay_disk, + image_disk, + kernel_image: args.vm_kernel_image.clone(), + vcpus: args.vm_vcpus, + mem_mib: args.vm_mem_mib, + exec_path, + args: Vec::new(), + env: args.vm_env.clone(), + workdir: args.vm_workdir.clone(), + log_level: args.vm_krun_log_level, + console_output, + backend, + gpu_bdf: args.vm_gpu_bdf.clone(), + tap_device: args.vm_tap_device.clone(), + guest_ip: args.vm_guest_ip.clone(), + host_ip: args.vm_host_ip.clone(), + vsock_cid: args.vm_vsock_cid, + guest_mac: args.vm_guest_mac.clone(), + gateway_port: args.vm_gateway_port, + }) +} + +#[cfg(target_os = "macos")] +fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { + use std::os::unix::process::CommandExt as _; + + const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; + + if std::env::var_os(REEXEC_ENV).is_some() { + return Ok(()); + } + + let runtime_dir = configured_runtime_dir().map_err(|err| miette::miette!("{err}"))?; + let runtime_str = runtime_dir.to_string_lossy(); + let needs_reexec = std::env::var_os("DYLD_LIBRARY_PATH") + .is_none_or(|value| !value.to_string_lossy().contains(runtime_str.as_ref())); + if !needs_reexec { + return Ok(()); + } + + let mut dyld_paths = vec![runtime_dir.clone()]; + if let Some(existing) = std::env::var_os("DYLD_LIBRARY_PATH") { + dyld_paths.extend(std::env::split_paths(&existing)); + } + let joined = std::env::join_paths(&dyld_paths) + .map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?; + let exe = std::env::current_exe().into_diagnostic()?; + let args: Vec = std::env::args().skip(1).collect(); + + // Use execvp() so the current process is *replaced* by the re-exec'd + // binary — no wrapper process sits between the compute driver and + // the actually-running VM launcher. That avoids two problems: + // 1. An extra process level that survives SIGKILL of the driver + // (the wrapper was reparenting the re-exec'd child to init). + // 2. Signal forwarding: with a wrapper, a SIGTERM to the wrapper + // doesn't reach the child unless we hand-roll forwarding. + // After exec, the child inherits our PID and our procguard arming. + let err = std::process::Command::new(exe) + .args(&args) + .env("DYLD_LIBRARY_PATH", &joined) + .env(VM_RUNTIME_DIR_ENV, runtime_dir) + .env(REEXEC_ENV, "1") + .exec(); + // `exec()` only returns on failure. + Err(miette::miette!("failed to re-exec with runtime env: {err}")) +} + +#[cfg(not(target_os = "macos"))] +// Signature must match the macOS variant which can fail. +#[allow(clippy::unnecessary_wraps)] +fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::{ + Args, ComputeDriverListenMode, PeerCredentials, authorize_peer_credentials, + compute_driver_listen_mode, + }; + use clap::Parser; + use std::path::PathBuf; + + #[test] + fn peer_authorization_accepts_matching_uid_and_pid() { + authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: Some(42), + }, + 1000, + Some(42), + ) + .unwrap(); + } + + #[test] + fn peer_authorization_rejects_wrong_pid() { + let err = authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: Some(7), + }, + 1000, + Some(42), + ) + .expect_err("wrong pid should be rejected"); + assert!(err.contains("does not match expected gateway pid")); + } + + #[test] + fn peer_authorization_rejects_wrong_uid() { + let err = authorize_peer_credentials( + PeerCredentials { + uid: 1001, + pid: Some(42), + }, + 1000, + Some(42), + ) + .expect_err("wrong uid should be rejected"); + assert!(err.contains("does not match current euid")); + } + + #[test] + fn peer_authorization_rejects_missing_pid_when_expected() { + let err = authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: None, + }, + 1000, + Some(42), + ) + .expect_err("missing pid should be rejected"); + assert!(err.contains("peer pid is unavailable")); + } + + #[test] + fn peer_authorization_accepts_matching_uid_without_expected_pid() { + authorize_peer_credentials( + PeerCredentials { + uid: 1000, + pid: None, + }, + 1000, + None, + ) + .unwrap(); + } + + #[test] + fn listen_mode_rejects_default_tcp() { + let args = Args::parse_from(["openshell-driver-vm"]); + let err = compute_driver_listen_mode(&args).expect_err("default TCP should be disabled"); + assert!(err.contains("--bind-socket is required")); + } + + #[test] + fn accepts_gateway_otlp_endpoint() { + let args = Args::try_parse_from([ + "openshell-driver-vm", + "--otlp-endpoint", + "http://127.0.0.1:4317", + ]); + assert!( + args.is_ok(), + "VM driver should accept the gateway OTLP endpoint" + ); + } + + #[test] + fn listen_mode_rejects_bind_address_without_tcp_opt_in() { + let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]); + let err = + compute_driver_listen_mode(&args).expect_err("TCP bind should require explicit opt-in"); + assert!(err.contains("--allow-unauthenticated-tcp")); + } + + #[test] + fn listen_mode_requires_bind_address_with_tcp_opt_in() { + let args = Args::parse_from(["openshell-driver-vm", "--allow-unauthenticated-tcp"]); + let err = + compute_driver_listen_mode(&args).expect_err("TCP opt-in should require an address"); + assert!(err.contains("--bind-address is required")); + } + + #[test] + fn listen_mode_accepts_explicit_unauthenticated_tcp() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--allow-unauthenticated-tcp", + "--bind-address", + "127.0.0.1:50061", + ]); + assert_eq!( + compute_driver_listen_mode(&args).unwrap(), + ComputeDriverListenMode::Tcp("127.0.0.1:50061".parse().unwrap()) + ); + } + + #[test] + fn listen_mode_requires_expected_peer_pid_for_uds() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--bind-socket", + "/tmp/compute-driver.sock", + ]); + let err = compute_driver_listen_mode(&args) + .expect_err("UDS should require gateway peer pid by default"); + assert!(err.contains("--expected-peer-pid is required")); + } + + #[test] + fn listen_mode_accepts_uds_with_expected_peer_pid() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--bind-socket", + "/tmp/compute-driver.sock", + "--expected-peer-pid", + "42", + ]); + assert_eq!( + compute_driver_listen_mode(&args).unwrap(), + ComputeDriverListenMode::Unix { + socket_path: PathBuf::from("/tmp/compute-driver.sock"), + expected_peer_pid: Some(42), + } + ); + } + + #[test] + fn listen_mode_accepts_explicit_same_uid_uds_dev_mode() { + let args = Args::parse_from([ + "openshell-driver-vm", + "--bind-socket", + "/tmp/compute-driver.sock", + "--allow-same-uid-peer", + ]); + assert_eq!( + compute_driver_listen_mode(&args).unwrap(), + ComputeDriverListenMode::Unix { + socket_path: PathBuf::from("/tmp/compute-driver.sock"), + expected_peer_pid: None, + } + ); + } +} diff --git a/crates/openshell-driver-vm/src/main_entry.rs b/crates/openshell-driver-vm/src/main_entry.rs new file mode 100644 index 000000000..432402121 --- /dev/null +++ b/crates/openshell-driver-vm/src/main_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("main.rs"); + +#[cfg(target_os = "windows")] +include!("main_win.rs"); diff --git a/crates/openshell-driver-vm/src/main_unix.rs b/crates/openshell-driver-vm/src/main_unix.rs deleted file mode 100644 index 949d4ce05..000000000 --- a/crates/openshell-driver-vm/src/main_unix.rs +++ /dev/null @@ -1,782 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use clap::Parser; -use futures::Stream; -use miette::{IntoDiagnostic, Result}; -use openshell_core::VERSION; -use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer; -use openshell_driver_vm::otel_tracing::compute_driver_rpc_layer; -#[cfg(target_os = "macos")] -use openshell_driver_vm::{VM_RUNTIME_DIR_ENV, configured_runtime_dir}; -use openshell_driver_vm::{VmBackend, VmDriver, VmDriverConfig, VmLaunchConfig, procguard, run_vm}; -use std::io; -use std::net::SocketAddr; -use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; -use std::path::{Path, PathBuf}; -use std::pin::Pin; -use std::task::{Context, Poll}; -use tokio::net::{UnixListener, UnixStream}; -use tracing::info; -use tracing_subscriber::EnvFilter; -use tracing_subscriber::prelude::*; - -#[derive(Parser, Debug)] -#[command(name = "openshell-driver-vm")] -#[command(version = VERSION)] -#[allow(clippy::struct_excessive_bools)] -struct Args { - #[arg(long, hide = true, default_value_t = false)] - internal_run_vm: bool, - - #[arg(long = "vm-root-disk", hide = true, alias = "vm-rootfs")] - vm_root_disk: Option, - - #[arg(long = "vm-overlay-disk", hide = true)] - vm_overlay_disk: Option, - - #[arg(long = "vm-image-disk", hide = true)] - vm_image_disk: Option, - - #[arg(long = "vm-kernel-image", hide = true)] - vm_kernel_image: Option, - - #[arg(long, hide = true)] - vm_exec: Option, - - #[arg(long, hide = true, default_value = "/")] - vm_workdir: String, - - #[arg(long, hide = true)] - vm_env: Vec, - - #[arg(long, hide = true)] - vm_console_output: Option, - - #[arg(long, hide = true, default_value_t = 2)] - vm_vcpus: u8, - - #[arg(long, hide = true, default_value_t = 2048)] - vm_mem_mib: u32, - - #[arg(long, hide = true, default_value_t = 1)] - vm_krun_log_level: u32, - - #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_BIND")] - bind_address: Option, - - #[arg(long, env = "OPENSHELL_COMPUTE_DRIVER_SOCKET")] - bind_socket: Option, - - #[arg(long, hide = true)] - expected_peer_pid: Option, - - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_UNAUTHENTICATED_TCP", - default_value_t = false - )] - allow_unauthenticated_tcp: bool, - - #[arg( - long, - env = "OPENSHELL_COMPUTE_DRIVER_ALLOW_SAME_UID_PEER", - default_value_t = false - )] - allow_same_uid_peer: bool, - - #[arg(long, env = "OPENSHELL_LOG_LEVEL", default_value = "info")] - log_level: String, - - #[arg(long, env = "OPENSHELL_OTLP_ENDPOINT")] - otlp_endpoint: Option, - - #[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")] - openshell_endpoint: Option, - - #[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")] - default_image: String, - - #[arg(long, env = "OPENSHELL_VM_BOOTSTRAP_IMAGE", default_value = "")] - bootstrap_image: String, - - #[arg( - long, - env = "OPENSHELL_VM_DRIVER_STATE_DIR", - default_value = "target/openshell-vm-driver" - )] - state_dir: PathBuf, - - #[arg(long = "guest-tls-ca", env = "OPENSHELL_VM_TLS_CA")] - guest_tls_ca: Option, - - #[arg(long = "guest-tls-cert", env = "OPENSHELL_VM_TLS_CERT")] - guest_tls_cert: Option, - - #[arg(long = "guest-tls-key", env = "OPENSHELL_VM_TLS_KEY")] - guest_tls_key: Option, - - #[arg(long, env = "OPENSHELL_VM_KRUN_LOG_LEVEL", default_value_t = 1)] - krun_log_level: u32, - - #[arg(long, env = "OPENSHELL_VM_DRIVER_VCPUS", default_value_t = 2)] - vcpus: u8, - - #[arg(long, env = "OPENSHELL_VM_DRIVER_MEM_MIB", default_value_t = 2048)] - mem_mib: u32, - - #[arg(long, env = "OPENSHELL_VM_OVERLAY_DISK_MIB", default_value_t = 4096)] - overlay_disk_mib: u64, - - #[arg(long, env = "OPENSHELL_VM_GPU")] - gpu: bool, - - #[arg(long, env = "OPENSHELL_VM_GPU_MEM_MIB", default_value_t = 8192)] - gpu_mem_mib: u32, - - #[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)] - gpu_vcpus: u8, - - #[arg(long, env = "OPENSHELL_VM_SANDBOX_UID")] - sandbox_uid: Option, - - #[arg(long, env = "OPENSHELL_VM_SANDBOX_GID")] - sandbox_gid: Option, - - #[arg(long, hide = true)] - vm_backend: Option, - - #[arg(long, hide = true)] - vm_gpu_bdf: Option, - - #[arg(long, hide = true)] - vm_tap_device: Option, - - #[arg(long, hide = true)] - vm_guest_ip: Option, - - #[arg(long, hide = true)] - vm_host_ip: Option, - - #[arg(long, hide = true)] - vm_vsock_cid: Option, - - #[arg(long, hide = true)] - vm_guest_mac: Option, - - #[arg(long, hide = true)] - vm_gateway_port: Option, -} - -#[tokio::main] -async fn main() -> Result<()> { - let args = Args::parse(); - if args.internal_run_vm { - // We intentionally defer procguard arming until `run_vm()` so - // that the only arm is the one that knows how to clean up - // gvproxy. Racing two watchers against the same parent-death - // event causes the bare arm's `exit(1)` to win, skipping the - // gvproxy cleanup and leaking the helper. The risk window - // before `run_vm` arms procguard is ~a few syscalls long - // (`build_vm_launch_config`, `configured_runtime_dir`), which - // is negligible next to the parent gRPC server's uptime. - maybe_reexec_internal_vm_with_runtime_env()?; - let config = build_vm_launch_config(&args).map_err(|err| miette::miette!("{err}"))?; - run_vm(&config).map_err(|err| miette::miette!("{err}"))?; - return Ok(()); - } - - let (tracer_provider, setup_error) = - openshell_driver_vm::otel_tracing::provider_for(args.otlp_endpoint.as_deref()); - tracing_subscriber::registry() - .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level))) - .with(tracing_subscriber::fmt::layer()) - .with( - tracer_provider - .as_ref() - .map(openshell_driver_vm::otel_tracing::layer), - ) - .init(); - if let Some(error) = setup_error { - tracing::error!(%error, "OTLP exporting could not be started"); - } else if let Some(endpoint) = &args.otlp_endpoint { - info!(endpoint, "OTLP exporting enabled"); - } - - let listen_mode = compute_driver_listen_mode(&args).map_err(|err| miette::miette!("{err}"))?; - - // Arm procguard so that if the gateway is killed (SIGKILL or crash) - // we also die. Without this the driver is reparented to init and - // keeps its per-sandbox VM launchers alive forever. Launchers have - // their own procguards (armed in `run_vm`) which cascade cleanup of - // gvproxy and the libkrun worker the moment this driver exits. - if let Err(err) = procguard::die_with_parent() { - tracing::warn!( - error = %err, - "procguard arm failed; gateway crashes may orphan this driver" - ); - } - - let driver = VmDriver::new(VmDriverConfig { - openshell_endpoint: args - .openshell_endpoint - .ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?, - state_dir: args.state_dir.clone(), - launcher_bin: None, - default_image: args.default_image.clone(), - bootstrap_image: args.bootstrap_image.clone(), - log_level: args.log_level.clone(), - krun_log_level: args.krun_log_level, - vcpus: args.vcpus, - mem_mib: args.mem_mib, - overlay_disk_mib: args.overlay_disk_mib, - guest_tls_ca: args.guest_tls_ca.clone(), - guest_tls_cert: args.guest_tls_cert.clone(), - guest_tls_key: args.guest_tls_key.clone(), - gpu_enabled: args.gpu, - gpu_mem_mib: args.gpu_mem_mib, - gpu_vcpus: args.gpu_vcpus, - sandbox_uid: args.sandbox_uid, - sandbox_gid: args.sandbox_gid, - }) - .await - .map_err(|err| miette::miette!("{err}"))?; - - let result = match listen_mode { - ComputeDriverListenMode::Unix { - socket_path, - expected_peer_pid, - } => { - prepare_compute_driver_socket(&socket_path).map_err(|err| miette::miette!("{err}"))?; - - info!(socket = %socket_path.display(), "Starting vm compute driver"); - let listener = UnixListener::bind(&socket_path).into_diagnostic()?; - restrict_socket_permissions(&socket_path).map_err(|err| miette::miette!("{err}"))?; - let result = tonic::transport::Server::builder() - .layer(compute_driver_rpc_layer()) - .add_service(ComputeDriverServer::new(driver)) - .serve_with_incoming_shutdown( - AuthenticatedUnixIncoming::new(listener, expected_peer_pid), - shutdown_signal(), - ) - .await - .into_diagnostic(); - let _ = std::fs::remove_file(&socket_path); - result - } - ComputeDriverListenMode::Tcp(bind_address) => { - info!(address = %bind_address, "Starting unauthenticated dev vm compute driver"); - tonic::transport::Server::builder() - .layer(compute_driver_rpc_layer()) - .add_service(ComputeDriverServer::new(driver)) - .serve_with_shutdown(bind_address, shutdown_signal()) - .await - .into_diagnostic() - } - }; - if let Some(provider) = &tracer_provider - && let Err(error) = provider.shutdown() - { - tracing::warn!(%error, "OTLP tracer provider shutdown failed"); - } - result -} - -async fn shutdown_signal() { - let mut terminate = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) - .expect("install SIGTERM handler"); - tokio::select! { - result = tokio::signal::ctrl_c() => { - if let Err(error) = result { - tracing::warn!(%error, "failed to listen for Ctrl-C"); - } - } - _ = terminate.recv() => {} - } - info!("Shutdown signal received; stopping vm compute driver"); -} - -#[derive(Debug, Clone, PartialEq, Eq)] -enum ComputeDriverListenMode { - Unix { - socket_path: PathBuf, - expected_peer_pid: Option, - }, - Tcp(SocketAddr), -} - -fn compute_driver_listen_mode(args: &Args) -> std::result::Result { - if let Some(socket_path) = args.bind_socket.clone() { - if args.expected_peer_pid.is_none() && !args.allow_same_uid_peer { - return Err( - "--expected-peer-pid is required with --bind-socket; use --allow-same-uid-peer only for local development" - .to_string(), - ); - } - return Ok(ComputeDriverListenMode::Unix { - socket_path, - expected_peer_pid: args.expected_peer_pid, - }); - } - - if !args.allow_unauthenticated_tcp { - return Err( - "--bind-socket is required; unauthenticated TCP mode is disabled unless --allow-unauthenticated-tcp is set for local development" - .to_string(), - ); - } - - let Some(bind_address) = args.bind_address else { - return Err("--bind-address is required with --allow-unauthenticated-tcp".to_string()); - }; - - Ok(ComputeDriverListenMode::Tcp(bind_address)) -} - -fn prepare_compute_driver_socket(socket_path: &Path) -> std::result::Result<(), String> { - let Some(parent) = socket_path.parent() else { - return Err(format!( - "vm compute driver socket path '{}' has no parent directory", - socket_path.display() - )); - }; - let expected_uid = current_euid(); - prepare_private_socket_dir(parent, expected_uid)?; - remove_stale_socket(socket_path, expected_uid) -} - -fn current_euid() -> u32 { - rustix::process::geteuid().as_raw() -} - -fn prepare_private_socket_dir( - socket_dir: &Path, - expected_uid: u32, -) -> std::result::Result<(), String> { - std::fs::create_dir_all(socket_dir) - .map_err(|err| format!("create socket dir {}: {err}", socket_dir.display()))?; - let metadata = std::fs::symlink_metadata(socket_dir) - .map_err(|err| format!("stat socket dir {}: {err}", socket_dir.display()))?; - let file_type = metadata.file_type(); - if file_type.is_symlink() { - return Err(format!( - "socket dir {} is a symlink; refusing to use it", - socket_dir.display() - )); - } - if !file_type.is_dir() { - return Err(format!( - "socket dir {} is not a directory", - socket_dir.display() - )); - } - if metadata.uid() != expected_uid { - return Err(format!( - "socket dir {} is owned by uid {} but current euid is {}", - socket_dir.display(), - metadata.uid(), - expected_uid - )); - } - std::fs::set_permissions(socket_dir, std::fs::Permissions::from_mode(0o700)) - .map_err(|err| format!("chmod socket dir {}: {err}", socket_dir.display())) -} - -fn remove_stale_socket(socket_path: &Path, expected_uid: u32) -> std::result::Result<(), String> { - let metadata = match std::fs::symlink_metadata(socket_path) { - Ok(metadata) => metadata, - Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()), - Err(err) => return Err(format!("stat socket {}: {err}", socket_path.display())), - }; - let file_type = metadata.file_type(); - if file_type.is_symlink() { - return Err(format!( - "socket {} is a symlink; refusing to remove it", - socket_path.display() - )); - } - if metadata.uid() != expected_uid { - return Err(format!( - "socket {} is owned by uid {} but current euid is {}", - socket_path.display(), - metadata.uid(), - expected_uid - )); - } - if !file_type.is_socket() { - return Err(format!( - "socket path {} exists but is not a Unix socket", - socket_path.display() - )); - } - std::fs::remove_file(socket_path) - .map_err(|err| format!("remove stale socket {}: {err}", socket_path.display())) -} - -fn restrict_socket_permissions(socket_path: &Path) -> std::result::Result<(), String> { - std::fs::set_permissions(socket_path, std::fs::Permissions::from_mode(0o600)) - .map_err(|err| format!("chmod socket {}: {err}", socket_path.display())) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct PeerCredentials { - uid: u32, - pid: Option, -} - -fn peer_credentials(stream: &UnixStream) -> std::result::Result { - let credentials = stream - .peer_cred() - .map_err(|err| format!("read peer credentials: {err}"))?; - Ok(PeerCredentials { - uid: credentials.uid(), - pid: credentials.pid(), - }) -} - -fn authorize_peer_credentials( - peer: PeerCredentials, - driver_uid: u32, - gateway_pid: Option, -) -> std::result::Result<(), String> { - if peer.uid != driver_uid { - return Err(format!( - "peer uid {} does not match current euid {}", - peer.uid, driver_uid - )); - } - let Some(gateway_pid) = gateway_pid else { - return Ok(()); - }; - let Some(peer_process_id) = peer.pid.and_then(|pid| u32::try_from(pid).ok()) else { - return Err(format!( - "peer pid is unavailable; expected gateway pid {gateway_pid}" - )); - }; - if peer_process_id != gateway_pid { - return Err(format!( - "peer pid {peer_process_id} does not match expected gateway pid {gateway_pid}" - )); - } - Ok(()) -} - -struct AuthenticatedUnixIncoming { - listener: UnixListener, - expected_uid: u32, - expected_peer_pid: Option, -} - -impl AuthenticatedUnixIncoming { - fn new(listener: UnixListener, expected_peer_pid: Option) -> Self { - Self { - listener, - expected_uid: current_euid(), - expected_peer_pid, - } - } -} - -impl Stream for AuthenticatedUnixIncoming { - type Item = io::Result; - - fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - loop { - match this.listener.poll_accept(cx) { - Poll::Ready(Ok((stream, _addr))) => { - let authorized = peer_credentials(&stream).and_then(|peer| { - authorize_peer_credentials(peer, this.expected_uid, this.expected_peer_pid) - }); - match authorized { - Ok(()) => return Poll::Ready(Some(Ok(stream))), - Err(err) => { - tracing::warn!( - error = %err, - "rejected vm compute driver UDS client" - ); - } - } - } - Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), - Poll::Pending => return Poll::Pending, - } - } - } -} - -fn build_vm_launch_config(args: &Args) -> std::result::Result { - let root_disk = args - .vm_root_disk - .clone() - .ok_or_else(|| "--vm-root-disk is required in internal VM mode".to_string())?; - let overlay_disk = args - .vm_overlay_disk - .clone() - .ok_or_else(|| "--vm-overlay-disk is required in internal VM mode".to_string())?; - let image_disk = args.vm_image_disk.clone(); - let exec_path = args - .vm_exec - .clone() - .ok_or_else(|| "--vm-exec is required in internal VM mode".to_string())?; - let console_output = args - .vm_console_output - .clone() - .ok_or_else(|| "--vm-console-output is required in internal VM mode".to_string())?; - - let backend = match args.vm_backend.as_deref() { - Some("qemu") => VmBackend::Qemu, - Some("libkrun") | None => VmBackend::Libkrun, - Some(other) => return Err(format!("unknown VM backend: {other}")), - }; - - Ok(VmLaunchConfig { - root_disk, - overlay_disk, - image_disk, - kernel_image: args.vm_kernel_image.clone(), - vcpus: args.vm_vcpus, - mem_mib: args.vm_mem_mib, - exec_path, - args: Vec::new(), - env: args.vm_env.clone(), - workdir: args.vm_workdir.clone(), - log_level: args.vm_krun_log_level, - console_output, - backend, - gpu_bdf: args.vm_gpu_bdf.clone(), - tap_device: args.vm_tap_device.clone(), - guest_ip: args.vm_guest_ip.clone(), - host_ip: args.vm_host_ip.clone(), - vsock_cid: args.vm_vsock_cid, - guest_mac: args.vm_guest_mac.clone(), - gateway_port: args.vm_gateway_port, - }) -} - -#[cfg(target_os = "macos")] -fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { - use std::os::unix::process::CommandExt as _; - - const REEXEC_ENV: &str = "__OPENSHELL_DRIVER_VM_REEXEC"; - - if std::env::var_os(REEXEC_ENV).is_some() { - return Ok(()); - } - - let runtime_dir = configured_runtime_dir().map_err(|err| miette::miette!("{err}"))?; - let runtime_str = runtime_dir.to_string_lossy(); - let needs_reexec = std::env::var_os("DYLD_LIBRARY_PATH") - .is_none_or(|value| !value.to_string_lossy().contains(runtime_str.as_ref())); - if !needs_reexec { - return Ok(()); - } - - let mut dyld_paths = vec![runtime_dir.clone()]; - if let Some(existing) = std::env::var_os("DYLD_LIBRARY_PATH") { - dyld_paths.extend(std::env::split_paths(&existing)); - } - let joined = std::env::join_paths(&dyld_paths) - .map_err(|err| miette::miette!("join DYLD_LIBRARY_PATH: {err}"))?; - let exe = std::env::current_exe().into_diagnostic()?; - let args: Vec = std::env::args().skip(1).collect(); - - // Use execvp() so the current process is *replaced* by the re-exec'd - // binary — no wrapper process sits between the compute driver and - // the actually-running VM launcher. That avoids two problems: - // 1. An extra process level that survives SIGKILL of the driver - // (the wrapper was reparenting the re-exec'd child to init). - // 2. Signal forwarding: with a wrapper, a SIGTERM to the wrapper - // doesn't reach the child unless we hand-roll forwarding. - // After exec, the child inherits our PID and our procguard arming. - let err = std::process::Command::new(exe) - .args(&args) - .env("DYLD_LIBRARY_PATH", &joined) - .env(VM_RUNTIME_DIR_ENV, runtime_dir) - .env(REEXEC_ENV, "1") - .exec(); - // `exec()` only returns on failure. - Err(miette::miette!("failed to re-exec with runtime env: {err}")) -} - -#[cfg(not(target_os = "macos"))] -// Signature must match the macOS variant which can fail. -#[allow(clippy::unnecessary_wraps)] -fn maybe_reexec_internal_vm_with_runtime_env() -> Result<()> { - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::{ - Args, ComputeDriverListenMode, PeerCredentials, authorize_peer_credentials, - compute_driver_listen_mode, - }; - use clap::Parser; - use std::path::PathBuf; - - #[test] - fn peer_authorization_accepts_matching_uid_and_pid() { - authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: Some(42), - }, - 1000, - Some(42), - ) - .unwrap(); - } - - #[test] - fn peer_authorization_rejects_wrong_pid() { - let err = authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: Some(7), - }, - 1000, - Some(42), - ) - .expect_err("wrong pid should be rejected"); - assert!(err.contains("does not match expected gateway pid")); - } - - #[test] - fn peer_authorization_rejects_wrong_uid() { - let err = authorize_peer_credentials( - PeerCredentials { - uid: 1001, - pid: Some(42), - }, - 1000, - Some(42), - ) - .expect_err("wrong uid should be rejected"); - assert!(err.contains("does not match current euid")); - } - - #[test] - fn peer_authorization_rejects_missing_pid_when_expected() { - let err = authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: None, - }, - 1000, - Some(42), - ) - .expect_err("missing pid should be rejected"); - assert!(err.contains("peer pid is unavailable")); - } - - #[test] - fn peer_authorization_accepts_matching_uid_without_expected_pid() { - authorize_peer_credentials( - PeerCredentials { - uid: 1000, - pid: None, - }, - 1000, - None, - ) - .unwrap(); - } - - #[test] - fn listen_mode_rejects_default_tcp() { - let args = Args::parse_from(["openshell-driver-vm"]); - let err = compute_driver_listen_mode(&args).expect_err("default TCP should be disabled"); - assert!(err.contains("--bind-socket is required")); - } - - #[test] - fn accepts_gateway_otlp_endpoint() { - let args = Args::try_parse_from([ - "openshell-driver-vm", - "--otlp-endpoint", - "http://127.0.0.1:4317", - ]); - assert!( - args.is_ok(), - "VM driver should accept the gateway OTLP endpoint" - ); - } - - #[test] - fn listen_mode_rejects_bind_address_without_tcp_opt_in() { - let args = Args::parse_from(["openshell-driver-vm", "--bind-address", "127.0.0.1:50061"]); - let err = - compute_driver_listen_mode(&args).expect_err("TCP bind should require explicit opt-in"); - assert!(err.contains("--allow-unauthenticated-tcp")); - } - - #[test] - fn listen_mode_requires_bind_address_with_tcp_opt_in() { - let args = Args::parse_from(["openshell-driver-vm", "--allow-unauthenticated-tcp"]); - let err = - compute_driver_listen_mode(&args).expect_err("TCP opt-in should require an address"); - assert!(err.contains("--bind-address is required")); - } - - #[test] - fn listen_mode_accepts_explicit_unauthenticated_tcp() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--allow-unauthenticated-tcp", - "--bind-address", - "127.0.0.1:50061", - ]); - assert_eq!( - compute_driver_listen_mode(&args).unwrap(), - ComputeDriverListenMode::Tcp("127.0.0.1:50061".parse().unwrap()) - ); - } - - #[test] - fn listen_mode_requires_expected_peer_pid_for_uds() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--bind-socket", - "/tmp/compute-driver.sock", - ]); - let err = compute_driver_listen_mode(&args) - .expect_err("UDS should require gateway peer pid by default"); - assert!(err.contains("--expected-peer-pid is required")); - } - - #[test] - fn listen_mode_accepts_uds_with_expected_peer_pid() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--bind-socket", - "/tmp/compute-driver.sock", - "--expected-peer-pid", - "42", - ]); - assert_eq!( - compute_driver_listen_mode(&args).unwrap(), - ComputeDriverListenMode::Unix { - socket_path: PathBuf::from("/tmp/compute-driver.sock"), - expected_peer_pid: Some(42), - } - ); - } - - #[test] - fn listen_mode_accepts_explicit_same_uid_uds_dev_mode() { - let args = Args::parse_from([ - "openshell-driver-vm", - "--bind-socket", - "/tmp/compute-driver.sock", - "--allow-same-uid-peer", - ]); - assert_eq!( - compute_driver_listen_mode(&args).unwrap(), - ComputeDriverListenMode::Unix { - socket_path: PathBuf::from("/tmp/compute-driver.sock"), - expected_peer_pid: None, - } - ); - } -} diff --git a/crates/openshell-driver-vm/src/main_win.rs b/crates/openshell-driver-vm/src/main_win.rs new file mode 100644 index 000000000..8463707ca --- /dev/null +++ b/crates/openshell-driver-vm/src/main_win.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + eprintln!("openshell-driver-vm is unsupported on Windows"); + std::process::exit(1); +} diff --git a/crates/openshell-prover/src/lib.rs b/crates/openshell-prover/src/lib.rs index 226705204..913045fe7 100644 --- a/crates/openshell-prover/src/lib.rs +++ b/crates/openshell-prover/src/lib.rs @@ -26,11 +26,11 @@ use policy::parse_policy; use queries::run_all_queries; use report::{render_compact, render_report}; -/// Run the prover end-to-end and return an exit code. +/// Run the prover end-to-end and return a result containing an exit code. /// -/// - `0` — pass (no critical/high findings, or all accepted) -/// - `1` — fail (critical or high findings present) -/// - `2` — input error +/// - `Ok(0)` — pass (no findings, or all accepted) +/// - `Ok(1)` — fail (one or more unaccepted findings present) +/// - `Err(_)` — input or registry loading error /// /// Binary and API capability registries are embedded at compile time. /// Pass `registry_dir` to override with a custom filesystem registry. @@ -105,10 +105,11 @@ mod tests { fn test_filesystem_policy() { let path = testdata_dir().join("policy.yaml"); let model = parse_policy(&path).expect("failed to parse policy"); - let readable = model.filesystem_policy.readable_paths(); + let readable = model.filesystem_policy.readable_paths(None); assert!(readable.contains(&"/usr".to_owned())); assert!(readable.contains(&"/sandbox".to_owned())); assert!(readable.contains(&"/tmp".to_owned())); + assert!(readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned())); } // 3. Workdir NOT included by default (matches runtime behavior). @@ -121,8 +122,9 @@ filesystem_policy: - /usr "; let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); + let readable = model.filesystem_policy.readable_paths(None); assert!(!readable.contains(&"/sandbox".to_owned())); + assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned())); } // 4. Workdir excluded when include_workdir: false. @@ -136,8 +138,9 @@ filesystem_policy: - /usr "; let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); + let readable = model.filesystem_policy.readable_paths(None); assert!(!readable.contains(&"/sandbox".to_owned())); + assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned())); } // 5. No duplicate when workdir already in read_write. @@ -152,12 +155,30 @@ filesystem_policy: - /tmp "; let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(); + let readable = model.filesystem_policy.readable_paths(Some("/sandbox")); let sandbox_count = readable.iter().filter(|p| *p == "/sandbox").count(); assert_eq!(sandbox_count, 1); } - // 6. End-to-end: testdata policy with a github credential in scope and a + // 6. A resolved non-default workdir does not replace an explicit path. + #[test] + fn test_include_workdir_preserves_explicit_sandbox_path() { + let yaml = r" +version: 1 +filesystem_policy: + include_workdir: true + read_write: + - /sandbox +"; + let model = policy::parse_policy_str(yaml).expect("parse"); + let readable = model + .filesystem_policy + .readable_paths(Some("/workspace/project")); + assert!(readable.contains(&"/sandbox".to_owned())); + assert!(readable.contains(&"/workspace/project".to_owned())); + } + + // 7. End-to-end: testdata policy with a github credential in scope and a // bypass-L7 binary (git) emits an `l7_bypass_credentialed` finding. // The prover output is categorical, not severity-graded. #[test] diff --git a/crates/openshell-prover/src/lib_unix.rs b/crates/openshell-prover/src/lib_unix.rs deleted file mode 100644 index 913045fe7..000000000 --- a/crates/openshell-prover/src/lib_unix.rs +++ /dev/null @@ -1,314 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! Formal policy verification for `OpenShell` sandboxes. -//! -//! Encodes sandbox policies, binary capabilities, and credential scopes as Z3 -//! SMT constraints, then checks reachability queries to detect data exfiltration -//! paths and write-bypass violations. - -pub mod accepted_risks; -pub mod credentials; -pub mod finding; -pub mod model; -pub mod policy; -pub mod queries; -pub mod registry; -pub mod report; - -use std::path::Path; - -use miette::Result; - -use accepted_risks::{apply_accepted_risks, load_accepted_risks}; -use model::build_model; -use policy::parse_policy; -use queries::run_all_queries; -use report::{render_compact, render_report}; - -/// Run the prover end-to-end and return a result containing an exit code. -/// -/// - `Ok(0)` — pass (no findings, or all accepted) -/// - `Ok(1)` — fail (one or more unaccepted findings present) -/// - `Err(_)` — input or registry loading error -/// -/// Binary and API capability registries are embedded at compile time. -/// Pass `registry_dir` to override with a custom filesystem registry. -pub fn prove( - policy_path: &str, - credentials_path: &str, - registry_dir: Option<&str>, - accepted_risks_path: Option<&str>, - compact: bool, -) -> Result { - let policy = parse_policy(Path::new(policy_path))?; - - let (credential_set, binary_registry) = match registry_dir { - Some(dir) => { - let dir = Path::new(dir); - ( - credentials::load_credential_set_from_dir(Path::new(credentials_path), dir)?, - registry::load_binary_registry_from_dir(dir)?, - ) - } - None => ( - credentials::load_credential_set_embedded(Path::new(credentials_path))?, - registry::load_embedded_binary_registry()?, - ), - }; - - let z3_model = build_model(policy, credential_set, binary_registry); - let mut findings = run_all_queries(&z3_model); - - if let Some(ar_path) = accepted_risks_path { - let accepted = load_accepted_risks(Path::new(ar_path))?; - findings = apply_accepted_risks(findings, &accepted); - } - - let exit_code = if compact { - render_compact(&findings, policy_path, credentials_path) - } else { - render_report(&findings, policy_path, credentials_path) - }; - - Ok(exit_code) -} - -// =========================================================================== -// Tests -// =========================================================================== - -#[cfg(test)] -mod tests { - use super::*; - use std::path::PathBuf; - - fn testdata_dir() -> PathBuf { - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata") - } - - // 1. Parse testdata/policy.yaml, verify structure. - #[test] - fn test_parse_policy() { - let path = testdata_dir().join("policy.yaml"); - let model = parse_policy(&path).expect("failed to parse policy"); - assert_eq!(model.version, 1); - assert!(model.network_policies.contains_key("github_api")); - let rule = &model.network_policies["github_api"]; - assert_eq!(rule.name, "github-api"); - assert_eq!(rule.endpoints.len(), 2); - assert!(rule.binaries.len() >= 4); - } - - // 2. Verify readable_paths. - #[test] - fn test_filesystem_policy() { - let path = testdata_dir().join("policy.yaml"); - let model = parse_policy(&path).expect("failed to parse policy"); - let readable = model.filesystem_policy.readable_paths(None); - assert!(readable.contains(&"/usr".to_owned())); - assert!(readable.contains(&"/sandbox".to_owned())); - assert!(readable.contains(&"/tmp".to_owned())); - assert!(readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned())); - } - - // 3. Workdir NOT included by default (matches runtime behavior). - #[test] - fn test_include_workdir_default() { - let yaml = r" -version: 1 -filesystem_policy: - read_only: - - /usr -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(None); - assert!(!readable.contains(&"/sandbox".to_owned())); - assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned())); - } - - // 4. Workdir excluded when include_workdir: false. - #[test] - fn test_include_workdir_false() { - let yaml = r" -version: 1 -filesystem_policy: - include_workdir: false - read_only: - - /usr -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(None); - assert!(!readable.contains(&"/sandbox".to_owned())); - assert!(!readable.contains(&policy::WORKDIR_PATH_SYMBOL.to_owned())); - } - - // 5. No duplicate when workdir already in read_write. - #[test] - fn test_include_workdir_no_duplicate() { - let yaml = r" -version: 1 -filesystem_policy: - include_workdir: true - read_write: - - /sandbox - - /tmp -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model.filesystem_policy.readable_paths(Some("/sandbox")); - let sandbox_count = readable.iter().filter(|p| *p == "/sandbox").count(); - assert_eq!(sandbox_count, 1); - } - - // 6. A resolved non-default workdir does not replace an explicit path. - #[test] - fn test_include_workdir_preserves_explicit_sandbox_path() { - let yaml = r" -version: 1 -filesystem_policy: - include_workdir: true - read_write: - - /sandbox -"; - let model = policy::parse_policy_str(yaml).expect("parse"); - let readable = model - .filesystem_policy - .readable_paths(Some("/workspace/project")); - assert!(readable.contains(&"/sandbox".to_owned())); - assert!(readable.contains(&"/workspace/project".to_owned())); - } - - // 7. End-to-end: testdata policy with a github credential in scope and a - // bypass-L7 binary (git) emits an `l7_bypass_credentialed` finding. - // The prover output is categorical, not severity-graded. - #[test] - fn test_findings_for_github_policy() { - use finding::category; - - let policy_path = testdata_dir().join("policy.yaml"); - let creds_path = testdata_dir().join("credentials.yaml"); - - let pol = parse_policy(&policy_path).expect("parse policy"); - let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(pol, cred_set, bin_reg); - let findings = run_all_queries(&z3_model); - - let categories: std::collections::HashSet<&str> = - findings.iter().map(|f| f.query.as_str()).collect(); - assert!( - categories.contains(category::L7_BYPASS_CREDENTIALED), - "expected l7_bypass_credentialed finding for bypass-L7 binary with credential in scope; \ - got categories: {categories:?}" - ); - // Every emitted category must be one of the four v1 categories. - let allowed: std::collections::HashSet<&str> = [ - category::LINK_LOCAL_REACH, - category::L7_BYPASS_CREDENTIALED, - category::CREDENTIAL_REACH_EXPANSION, - category::CAPABILITY_EXPANSION, - ] - .into_iter() - .collect(); - for c in &categories { - assert!( - allowed.contains(*c), - "unexpected category {c} emitted by the prover" - ); - } - } - - #[test] - fn test_wildcard_endpoint_covering_credential_host_emits_credential_reach() { - use finding::{FindingPath, category}; - - let policy = policy::parse_policy_str( - r#" -version: 1 -network_policies: - github_wildcard: - name: github-wildcard - endpoints: - - host: "*.github.com" - port: 443 - protocol: rest - enforcement: enforce - access: read-write - binaries: - - path: /usr/bin/curl -"#, - ) - .expect("parse policy"); - let cred_set = - credentials::load_credential_set_embedded(&testdata_dir().join("credentials.yaml")) - .expect("load creds"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(policy, cred_set, bin_reg); - let findings = run_all_queries(&z3_model); - - let reach = findings - .iter() - .find(|finding| finding.query == category::CREDENTIAL_REACH_EXPANSION) - .expect("wildcard host covering api.github.com must be credentialed"); - assert!(reach.paths.iter().any(|path| matches!( - path, - FindingPath::Exfil(exfil) - if exfil.endpoint_host == "*.github.com" && exfil.binary == "/usr/bin/curl" - ))); - } - - #[test] - fn test_known_metadata_hostname_emits_link_local_finding() { - use finding::{FindingPath, category}; - - let policy = policy::parse_policy_str( - r" -version: 1 -network_policies: - metadata: - name: metadata - endpoints: - - host: metadata.google.internal - port: 80 - binaries: - - path: /usr/bin/curl -", - ) - .expect("parse policy"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(policy, credentials::CredentialSet::default(), bin_reg); - let findings = run_all_queries(&z3_model); - - let link_local = findings - .iter() - .find(|finding| finding.query == category::LINK_LOCAL_REACH) - .expect("known metadata hostname must emit link-local/metadata finding"); - assert!(link_local.paths.iter().any(|path| matches!( - path, - FindingPath::Exfil(exfil) - if exfil.endpoint_host == "metadata.google.internal" - ))); - } - - // 7. Empty policy produces no findings. - #[test] - fn test_empty_policy_no_findings() { - let policy_path = testdata_dir().join("empty-policy.yaml"); - let creds_path = testdata_dir().join("credentials.yaml"); - - let pol = parse_policy(&policy_path).expect("parse policy"); - let cred_set = credentials::load_credential_set_embedded(&creds_path).expect("load creds"); - let bin_reg = registry::load_embedded_binary_registry().expect("load registry"); - - let z3_model = build_model(pol, cred_set, bin_reg); - let findings = run_all_queries(&z3_model); - - assert!( - findings.is_empty(), - "deny-all policy should produce no findings, got: {findings:?}" - ); - } -} diff --git a/crates/openshell-sandbox/Cargo.toml b/crates/openshell-sandbox/Cargo.toml index 1c9a17f84..58b4cb006 100644 --- a/crates/openshell-sandbox/Cargo.toml +++ b/crates/openshell-sandbox/Cargo.toml @@ -3,6 +3,7 @@ [package] name = "openshell-sandbox" +autobins = false description = "OpenShell process sandbox and monitor" version.workspace = true edition.workspace = true @@ -10,9 +11,12 @@ rust-version.workspace = true license.workspace = true repository.workspace = true +[lib] +path = "src/lib_entry.rs" + [[bin]] name = "openshell-sandbox" -path = "src/main.rs" +path = "src/main_entry.rs" [target.'cfg(not(target_os = "windows"))'.dependencies] openshell-core = { path = "../openshell-core", default-features = false } diff --git a/crates/openshell-sandbox/src/lib_entry.rs b/crates/openshell-sandbox/src/lib_entry.rs new file mode 100644 index 000000000..4272e7c53 --- /dev/null +++ b/crates/openshell-sandbox/src/lib_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("lib.rs"); + +#[cfg(target_os = "windows")] +include!("lib_win.rs"); diff --git a/crates/openshell-sandbox/src/lib_unix.rs b/crates/openshell-sandbox/src/lib_unix.rs deleted file mode 100644 index e696403f3..000000000 --- a/crates/openshell-sandbox/src/lib_unix.rs +++ /dev/null @@ -1,3773 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! `OpenShell` Sandbox library. -//! -//! This crate provides process sandboxing and monitoring capabilities. - -mod activity_aggregator; -mod denial_aggregator; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod google_cloud_metadata; -mod mechanistic_mapper; -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -mod metadata_server; -mod sidecar_control; - -use miette::{IntoDiagnostic, Result, WrapErr}; -use std::future::Future; -use std::sync::Arc; -#[cfg(target_os = "linux")] -use std::sync::atomic::Ordering; -use std::sync::atomic::{AtomicBool, AtomicU32}; -use std::time::Duration; -use tracing::{debug, info, warn}; - -use openshell_ocsf::{ - ActionId, ActivityId, AppLifecycleBuilder, ConfigStateChangeBuilder, DetectionFindingBuilder, - DispositionId, FindingInfo, SandboxContext, SeverityId, StateId, StatusId, ocsf_emit, -}; - -// --------------------------------------------------------------------------- -// OCSF Context -// --------------------------------------------------------------------------- -// -// The following log sites intentionally remain as plain `tracing` macros -// and are NOT migrated to OCSF builders: -// -// - DEBUG/TRACE events (zombie reaping, ip commands, gRPC connects, PTY state) -// - Transient "about to do X" events where the result is logged separately -// (e.g., "Fetching sandbox policy via gRPC", "Creating OPA engine from proto") -// - Internal SSH channel warnings (unknown channel, PTY resize failures) -// - Denial flush telemetry (the individual denials are already OCSF events) -// - Status reporting failures (sync to gateway, non-actionable) -// - Route refresh interval validation warnings -// -// These are operational plumbing that don't represent security decisions, -// policy changes, or observable sandbox behavior worth structuring. -// --------------------------------------------------------------------------- - -/// Re-export the process-wide OCSF sandbox context getter. -/// -/// The singleton lives in `openshell-ocsf` so both supervisor leaves can -/// reach it without depending on `openshell-sandbox`. Initialised once during -/// `run_sandbox()` startup via `openshell_ocsf::ctx::set_ctx`. -pub(crate) use openshell_ocsf::ctx::ctx as ocsf_ctx; - -use openshell_core::denial::DenialEvent; -use openshell_core::policy::{NetworkMode, NetworkPolicy, ProxyPolicy, SandboxPolicy}; -use openshell_core::proposals::AgentProposals; -use openshell_core::provider_credentials::ProviderCredentialState; -use openshell_supervisor_network::opa::OpaEngine; -use openshell_supervisor_process::process::ProcessEnforcementMode; -pub use openshell_supervisor_process::process::{ProcessHandle, ProcessStatus}; -use openshell_supervisor_process::skills; -use tokio::sync::mpsc::UnboundedSender; -#[cfg(any(test, target_os = "linux"))] -use tokio::time::timeout; - -const SIDECAR_NETWORK_ENFORCEMENT_MODE: &str = "sidecar-nftables"; -const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy"; -const SIDECAR_CA_CERT: &str = "openshell-ca.pem"; -const SIDECAR_CA_BUNDLE: &str = "ca-bundle.pem"; -const SIDECAR_PROCESS_PROXY_ADDR: &str = "127.0.0.1:3128"; -const SIDECAR_READY_TIMEOUT_SECS: u64 = 120; - -/// Run a command in the sandbox. -/// -/// # Errors -/// -/// Returns an error if the command fails to start or encounters a fatal error. -#[allow( - clippy::too_many_arguments, - clippy::similar_names, - clippy::fn_params_excessive_bools -)] -pub async fn run_sandbox( - command: Vec, - workdir: Option, - timeout_secs: u64, - interactive: bool, - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, - ssh_socket_path: Option, - _health_check: bool, - _health_port: u16, - inference_routes: Option, - ocsf_enabled: Arc, - network_enabled: bool, - process_enabled: bool, - upstream_proxy_args: openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs, -) -> Result { - let (program, args) = command - .split_first() - .ok_or_else(|| miette::miette!("No command specified"))?; - - // Initialize the process-wide OCSF context early so that events emitted - // during policy loading (filesystem config, validation) have a context. - // Proxy IP/port use defaults here; they are only significant for network - // events which happen after the netns is created. - { - let hostname = std::fs::read_to_string("/etc/hostname").map_or_else( - |_| "openshell-sandbox".to_string(), - |s| s.trim().to_string(), - ); - - if !openshell_ocsf::ctx::set_ctx(SandboxContext { - sandbox_id: sandbox_id.clone().unwrap_or_default(), - sandbox_name: sandbox.as_deref().unwrap_or_default().to_string(), - container_image: std::env::var("OPENSHELL_CONTAINER_IMAGE").unwrap_or_default(), - hostname, - product_version: openshell_core::VERSION.to_string(), - proxy_ip: std::net::IpAddr::from([127, 0, 0, 1]), - proxy_port: 3128, - }) { - debug!("OCSF context already initialized, keeping existing"); - } - } - - let sidecar_network_enforcement = sidecar_network_enforcement_enabled(); - let process_enforcement_mode = process_enforcement_mode(); - let process_uses_sidecar_control = - process_enabled && !network_enabled && sidecar_network_enforcement; - let mut process_control_connection = None; - let sidecar_bootstrap = if process_uses_sidecar_control { - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for process-only sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let (bootstrap, connection) = sidecar_control::connect_process_client( - &socket, - Duration::from_secs(SIDECAR_READY_TIMEOUT_SECS), - ) - .await?; - process_control_connection = Some(connection); - Some(bootstrap) - } else { - None - }; - - // Load policy and initialize OPA engine - let openshell_endpoint_for_proxy = openshell_endpoint.clone(); - let sandbox_name_for_agg = sandbox.clone(); - let ( - mut policy, - opa_engine, - retained_proto, - middleware_registry_status, - loaded_policy_origin, - initial_agent_proposals_enabled, - ) = if let Some(bootstrap) = sidecar_bootstrap.as_ref() { - let (policy, opa_engine, retained_proto, loaded_policy_origin) = - load_policy_from_sidecar_bootstrap(bootstrap)?; - ( - policy, - opa_engine, - retained_proto, - MiddlewareRegistryStatus::Synchronized, - loaded_policy_origin, - bootstrap.agent_proposals_enabled, - ) - } else { - load_policy( - sandbox_id.clone(), - sandbox, - openshell_endpoint.clone(), - policy_rules, - policy_data, - ) - .await? - }; - - // Override the policy's process identity with the driver-resolved UID/GID - // from the pod environment. The policy defaults to the name "sandbox" which - // resolves via /etc/passwd, but the driver may have chosen a different - // numeric UID (e.g. from OpenShift SCC annotations). - // Validate overrides against the same rules as the policy layer to prevent - // env-injected values (e.g. GID 0) from bypassing policy restrictions. - if let Ok(uid) = std::env::var(openshell_core::sandbox_env::SANDBOX_UID) - && !uid.is_empty() - { - if !openshell_policy::is_valid_sandbox_identity(&uid) { - return Err(miette::miette!( - "OPENSHELL_SANDBOX_UID contains invalid sandbox identity '{uid}'; \ - expected 'sandbox' or a numeric UID in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID, - )); - } - policy.process.run_as_user = Some(uid); - } - if let Ok(gid) = std::env::var(openshell_core::sandbox_env::SANDBOX_GID) - && !gid.is_empty() - { - if !openshell_policy::is_valid_sandbox_identity(&gid) { - return Err(miette::miette!( - "OPENSHELL_SANDBOX_GID contains invalid sandbox identity '{gid}'; \ - expected 'sandbox' or a numeric GID in range [{}, {}]", - openshell_policy::MIN_SANDBOX_UID, - openshell_policy::MAX_SANDBOX_UID, - )); - } - policy.process.run_as_group = Some(gid); - } - - #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] - let (provider_credentials, mut provider_env) = - if let Some(bootstrap) = sidecar_bootstrap.as_ref() { - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - bootstrap.provider_env_revision, - bootstrap.provider_child_env.clone(), - ); - (provider_credentials, bootstrap.provider_child_env.clone()) - } else { - // Fetch provider environment variables from the server. - // This is done after loading the policy so the sandbox can still start - // even if provider env fetch fails (graceful degradation). - let ( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ) = if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - match openshell_core::grpc_client::fetch_provider_environment(endpoint, id).await { - Ok(result) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Fetched provider environment [env_count:{}]", - result.environment.len() - )) - .build() - ); - ( - result.provider_env_revision, - result.environment, - result.credential_expires_at_ms, - result.dynamic_credentials, - ) - } - Err(e) => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .message(format!( - "Failed to fetch provider environment, continuing without: {e}" - )) - .build() - ); - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - } - } - } else { - ( - 0, - std::collections::HashMap::new(), - std::collections::HashMap::new(), - std::collections::HashMap::new(), - ) - }; - - let provider_credentials = ProviderCredentialState::from_environment( - provider_env_revision, - provider_env, - provider_credential_expires_at_ms, - dynamic_credentials, - ); - let provider_env = provider_credentials.child_env_with_gcp_resolved(); - (provider_credentials, provider_env) - }; - - // Shared agent-proposals feature flag. Seed from the same initial settings - // snapshot that produced the policy so networking and process setup agree - // before the poll loop starts reconciling later changes. - let agent_proposals = AgentProposals::new(initial_agent_proposals_enabled); - - let process_control_writer = process_control_connection - .as_ref() - .map(|connection| connection.writer.clone()); - let mut process_control_closed = None; - if let Some(connection) = process_control_connection { - process_control_closed = Some(connection.closed); - spawn_sidecar_control_update_watcher( - connection.updates, - provider_credentials.clone(), - agent_proposals.clone(), - ); - } - - // Shared PID: set after process spawn so the proxy can look up - // the entrypoint process's /proc/net/tcp for identity binding. - let entrypoint_pid = Arc::new(AtomicU32::new(0)); - - // Create the workload's network namespace. It is shared infrastructure: - // the proxy binds to its host-side veth IP, the bypass monitor reads - // /dev/kmsg from inside it, and the workload child / SSH sessions enter - // it via setns(). The RAII handle lives in this frame for the duration - // of the sandbox. - #[cfg(target_os = "linux")] - let netns = if network_enabled && !sidecar_network_enforcement { - openshell_supervisor_process::netns::create_netns_for_proxy(&policy)? - } else { - None - }; - - // The denial channel is owned by the orchestrator: the proxy (in the - // networking leaf) and the bypass monitor (in the process leaf) both - // produce DenialEvents that the denial aggregator (orchestrator-side) - // consumes via the matching receiver. Both leaves are pure producers; - // the orchestrator owns the consumer task spawned below. - let (denial_tx, denial_rx, bypass_denial_tx): ( - Option>, - _, - Option>, - ) = if sandbox_id.is_some() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_denial_tx); - - // Anonymous activity channel: same orchestrator-owned pattern as the - // denial channel. The proxy and the bypass monitor both emit per-event - // activity records; the orchestrator-side aggregator drains, sanitizes, - // and flushes anonymous summaries to the gateway. - let (activity_tx, activity_rx, bypass_activity_tx) = if sandbox_id.is_some() { - let (tx, rx) = - tokio::sync::mpsc::channel(openshell_core::activity::ACTIVITY_EVENT_QUEUE_CAPACITY); - let bypass_tx = tx.clone(); - (Some(tx), Some(rx), Some(bypass_tx)) - } else { - (None, None, None) - }; - #[cfg(not(target_os = "linux"))] - drop(bypass_activity_tx); - - // Workspace watch: the policy poll loop learns the workspace from - // GetSandboxConfig and broadcasts it. Flush tasks and the policy.local - // API read the current value so proposals target the correct workspace. - let (workspace_tx, workspace_rx) = tokio::sync::watch::channel(String::new()); - - let networking = if network_enabled { - #[cfg(target_os = "linux")] - let proxy_bind_ip = netns - .as_ref() - .map(openshell_supervisor_process::netns::NetworkNamespace::host_ip); - #[cfg(not(target_os = "linux"))] - let proxy_bind_ip: Option = None; - - Some( - openshell_supervisor_network::run::run_networking( - &policy, - proxy_bind_ip, - opa_engine.as_ref(), - retained_proto.as_ref(), - entrypoint_pid.clone(), - process_enabled, - &provider_credentials, - sandbox_id.as_deref(), - sandbox_name_for_agg.as_deref(), - openshell_endpoint_for_proxy.as_deref(), - inference_routes.as_deref(), - denial_tx, - activity_tx, - agent_proposals.clone(), - workspace_rx.clone(), - &upstream_proxy_args, - ) - .await?, - ) - } else { - None - }; - - #[cfg(target_os = "linux")] - let sidecar_control_server = if network_enabled && sidecar_network_enforcement { - if !matches!(policy.network.mode, NetworkMode::Proxy) { - return Err(miette::miette!( - "sidecar network enforcement requires proxy network mode" - )); - } - let socket = sidecar_control_socket().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar topology", - openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET - ) - })?; - let proto = retained_proto.as_ref().ok_or_else(|| { - miette::miette!( - "sidecar topology requires gateway policy data for the process supervisor" - ) - })?; - let ca_paths = networking.as_ref().and_then(|n| n.ca_file_paths.clone()); - Some(sidecar_control::spawn_server( - &socket, - sidecar_control::BootstrapData { - policy_proto: proto.clone(), - provider_env_revision: provider_credentials.snapshot().revision, - provider_child_env: provider_env.clone(), - agent_proposals_enabled: agent_proposals.enabled(), - proxy_ca_cert_path: ca_paths.as_ref().map(|paths| paths.0.clone()), - proxy_ca_bundle_path: ca_paths.as_ref().map(|paths| paths.1.clone()), - }, - sidecar_expected_peer()?, - )?) - } else { - None - }; - #[cfg(not(target_os = "linux"))] - let sidecar_control_server: Option = None; - - let sidecar_control_publisher = sidecar_control_server - .as_ref() - .map(sidecar_control::ServerHandle::publisher); - - #[cfg(target_os = "linux")] - let mut sidecar_control_task = None; - - #[cfg(target_os = "linux")] - if network_enabled - && sidecar_network_enforcement - && let Some(server) = sidecar_control_server - { - let trusted_ssh_socket_path = ssh_socket_path.clone().ok_or_else(|| { - miette::miette!( - "{} is required for sidecar network topology", - openshell_core::sandbox_env::SSH_SOCKET_PATH - ) - })?; - let (entrypoint_rx, connection_task) = server.into_runtime_parts(); - sidecar_control_task = Some(connection_task); - spawn_sidecar_entrypoint_handler( - entrypoint_rx, - entrypoint_pid.clone(), - opa_engine.clone(), - retained_proto.clone(), - openshell_endpoint.clone(), - sandbox_id.clone(), - std::path::PathBuf::from(trusted_ssh_socket_path), - ); - } - - #[cfg(not(target_os = "linux"))] - if network_enabled && sidecar_network_enforcement { - return Err(miette::miette!( - "sidecar network enforcement is only supported on Linux" - )); - } - - // Spawn the denial-aggregator flush task. The aggregator drains denial - // events from the proxy + bypass monitor, batches them, and ships - // summaries to the gateway via `SubmitPolicyAnalysis`. - if let (Some(rx), Some(endpoint)) = (denial_rx, openshell_endpoint_for_proxy.as_deref()) { - // SubmitPolicyAnalysis resolves by sandbox *name*, not UUID — fall - // back to the ID when the name isn't set. - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs: u64 = std::env::var("OPENSHELL_DENIAL_FLUSH_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - - let aggregator = denial_aggregator::DenialAggregator::new(rx, flush_interval_secs); - let denial_workspace_gate = workspace_rx.clone(); - let denial_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - |summaries| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = denial_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_proposals_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summaries, - ) - .await - { - warn!(error = %e, "Failed to flush denial summaries to gateway"); - } - } - }, - move || !denial_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn the activity-aggregator flush task. The aggregator drains - // anonymous activity events from the proxy, sanitizes deny groups, - // and ships periodic summaries to the gateway. - if let (Some(rx), Some(endpoint)) = (activity_rx, openshell_endpoint_for_proxy.as_deref()) { - let agg_name = sandbox_name_for_agg - .clone() - .or_else(|| sandbox_id.clone()) - .unwrap_or_default(); - let agg_endpoint = endpoint.to_string(); - let flush_interval_secs = activity_aggregator::activity_flush_interval_secs_from_env( - std::env::var("OPENSHELL_ACTIVITY_FLUSH_INTERVAL_SECS") - .ok() - .as_deref(), - ); - - let aggregator = activity_aggregator::ActivityAggregator::new(rx, flush_interval_secs); - let activity_workspace_gate = workspace_rx.clone(); - let activity_workspace_rx = workspace_rx.clone(); - - tokio::spawn(async move { - aggregator - .run( - move |summary| { - let endpoint = agg_endpoint.clone(); - let sandbox_name = agg_name.clone(); - let workspace = activity_workspace_rx.borrow().clone(); - async move { - if let Err(e) = flush_activity_to_gateway( - &endpoint, - &sandbox_name, - &workspace, - summary, - ) - .await - { - warn!(error = %e, "Failed to flush activity summary to gateway"); - } - } - }, - move || !activity_workspace_gate.borrow().is_empty(), - ) - .await; - }); - } - - // Spawn background policy poll task (gRPC mode only). - if !process_uses_sidecar_control - && let (Some(id), Some(endpoint), Some(engine)) = ( - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - opa_engine.as_ref(), - ) - { - let poll_id = id.to_string(); - let poll_endpoint = endpoint.to_string(); - let poll_engine = engine.clone(); - let poll_ocsf_enabled = ocsf_enabled.clone(); - let poll_pid = entrypoint_pid.clone(); - let poll_provider_credentials = provider_credentials.clone(); - let poll_policy_local = networking.as_ref().map(|n| n.policy_local_ctx.clone()); - let poll_interval_secs: u64 = std::env::var("OPENSHELL_POLICY_POLL_INTERVAL_SECS") - .ok() - .and_then(|v| v.parse().ok()) - .unwrap_or(10); - let poll_ctx = PolicyPollLoopContext { - endpoint: poll_endpoint, - sandbox_id: poll_id, - opa_engine: poll_engine, - loaded_policy_origin, - entrypoint_pid: poll_pid, - interval_secs: poll_interval_secs, - ocsf_enabled: poll_ocsf_enabled, - provider_credentials: poll_provider_credentials, - policy_local_ctx: poll_policy_local, - agent_proposals: agent_proposals.clone(), - middleware_registry_status, - sidecar_control_publisher: sidecar_control_publisher.clone(), - workspace_tx, - }; - - tokio::spawn(async move { - if let Err(e) = run_policy_poll_loop(poll_ctx).await { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .message(format!("Policy poll loop exited with error: {e}")) - .build() - ); - } - }); - } - - // Start GCE metadata loopback server inside the network namespace so - // Go's cloud.google.com/go/compute/metadata (which bypasses HTTP_PROXY) - // can reach it via direct TCP. Must start before the process leaf so SSH - // sessions also see corrected env vars on bind failure. - #[cfg(target_os = "linux")] - if let Some(ns) = netns.as_ref() - && provider_credentials - .snapshot() - .child_env - .contains_key("GCE_METADATA_HOST") - { - let ctx = google_cloud_metadata::MetadataContext::new(provider_credentials.clone()); - let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); - match ns - .bind_tcp_in_netns(openshell_core::google_cloud::METADATA_LOOPBACK_ADDR) - .await - { - Ok(listener) => { - tokio::spawn(metadata_server::run(listener, ctx, ready_tx)); - if let Ok(Ok(addr)) = timeout(Duration::from_secs(5), ready_rx).await { - info!(addr = %addr, "GCE metadata loopback server ready"); - } else { - warn!("GCE metadata server failed to become ready, removing metadata env vars"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - Err(e) => { - warn!(error = %e, "GCE metadata server bind failed, Go SDK may not discover credentials"); - provider_env.remove("GCE_METADATA_HOST"); - provider_env.remove("GCE_METADATA_IP"); - provider_env.remove("METADATA_SERVER_DETECTION"); - provider_credentials.remove_env_key("GCE_METADATA_HOST"); - } - } - } - - let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; - let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { - bootstrap - .proxy_ca_cert_path - .clone() - .zip(bootstrap.proxy_ca_bundle_path.clone()) - }); - - let exit_code = if process_enabled { - let ca_file_paths = networking - .as_ref() - .and_then(|n| n.ca_file_paths.clone()) - .or_else(|| { - if sidecar_network_enforcement { - sidecar_bootstrap_ca_file_paths - .clone() - .or_else(sidecar_ca_file_paths) - } else { - None - } - }); - - let entrypoint_started_tx = - if process_uses_sidecar_control && let Some(writer) = process_control_writer.clone() { - let (tx, rx) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - match rx.await { - Ok(pid) => { - if let Err(err) = - sidecar_control::send_entrypoint_started(&writer, pid).await - { - warn!(error = %err, "Failed to send sidecar entrypoint event"); - } - } - Err(_closed) => { - debug!("Entrypoint exited before sidecar entrypoint event was sent"); - } - } - }); - Some(tx) - } else { - None - }; - - let process = openshell_supervisor_process::run::run_process( - program, - args, - workdir.as_deref(), - timeout_secs, - interactive, - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - ssh_socket_path, - sidecar_network_enforcement, - &process_policy, - process_enforcement_mode, - entrypoint_pid, - entrypoint_started_tx, - provider_credentials, - provider_env, - ca_file_paths, - agent_proposals.clone(), - #[cfg(target_os = "linux")] - netns.as_ref(), - #[cfg(target_os = "linux")] - bypass_denial_tx, - #[cfg(target_os = "linux")] - bypass_activity_tx, - ); - - if let Some(control_closed) = process_control_closed.as_mut() { - tokio::select! { - result = process => result?, - _ = control_closed => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Authoritative network-sidecar control channel closed; terminating process container" - ) - .build() - ); - return Err(miette::miette!( - "authoritative network-sidecar control channel closed" - )); - } - } - } else { - process.await? - } - } else { - // Network-only sidecar mode: keep the proxy and its background - // tasks alive (held via the `networking` value) until shutdown. If the - // sole authenticated process-supervisor control connection closes, - // exit non-zero so Kubernetes restarts the network sidecar and creates - // a fresh one-client bootstrap listener for the restarted agent. - #[cfg(target_os = "linux")] - if let Some(control_task) = sidecar_control_task { - tokio::select! { - () = wait_for_shutdown_signal() => 0, - result = control_task => { - warn!(?result, "Authoritative sidecar control channel exited; restarting sidecar"); - 1 - } - } - } else { - wait_for_shutdown_signal().await; - 0 - } - #[cfg(not(target_os = "linux"))] - { - wait_for_shutdown_signal().await; - 0 - } - }; - - // Drop networking explicitly so the proxy + bypass monitor RAII - // handles tear down before we return. - drop(networking); - - Ok(exit_code) -} - -/// Wait for SIGINT or SIGTERM. Used in network-only mode where there is -/// no entrypoint child whose lifetime drives the supervisor's exit. -async fn wait_for_shutdown_signal() { - #[cfg(unix)] - { - use tokio::signal::unix::{SignalKind, signal}; - let mut sigterm = match signal(SignalKind::terminate()) { - Ok(s) => s, - Err(e) => { - tracing::warn!( - error = %e, - "Failed to install SIGTERM handler; waiting on SIGINT only" - ); - let _ = tokio::signal::ctrl_c().await; - return; - } - }; - tokio::select! { - _ = tokio::signal::ctrl_c() => { - info!("Received SIGINT, shutting down network-only supervisor"); - } - _ = sigterm.recv() => { - info!("Received SIGTERM, shutting down network-only supervisor"); - } - } - } - #[cfg(not(unix))] - { - let _ = tokio::signal::ctrl_c().await; - info!("Received Ctrl-C, shutting down network-only supervisor"); - } -} - -fn sidecar_network_enforcement_enabled() -> bool { - std::env::var(openshell_core::sandbox_env::NETWORK_ENFORCEMENT_MODE) - .is_ok_and(|value| value == SIDECAR_NETWORK_ENFORCEMENT_MODE) -} - -fn process_enforcement_mode() -> ProcessEnforcementMode { - match std::env::var(openshell_core::sandbox_env::SUPERVISOR_TOPOLOGY) - .ok() - .as_deref() - { - Some("sidecar") => ProcessEnforcementMode::NetworkOnly, - _ => ProcessEnforcementMode::Full, - } -} - -fn sidecar_control_socket() -> Option { - std::env::var(openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET) - .ok() - .filter(|path| !path.is_empty()) - .map(std::path::PathBuf::from) -} - -#[cfg_attr(not(target_os = "linux"), allow(dead_code))] -fn sidecar_expected_peer() -> Result { - fn required_numeric_env(name: &str) -> Result { - let value = std::env::var(name) - .into_diagnostic() - .wrap_err_with(|| format!("{name} is required for sidecar control authentication"))?; - value.parse::().into_diagnostic().wrap_err_with(|| { - format!("{name} must be a numeric ID for sidecar control authentication") - }) - } - - Ok(sidecar_control::ExpectedPeer { - uid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_UID)?, - gid: required_numeric_env(openshell_core::sandbox_env::SANDBOX_GID)?, - }) -} - -type LoadedPolicyBundle = ( - SandboxPolicy, - Option>, - Option, - LoadedPolicyOrigin, -); - -fn load_policy_from_sidecar_bootstrap( - bootstrap: &sidecar_control::BootstrapData, -) -> Result { - let proto = bootstrap.policy_proto.clone(); - let opa_engine = Some(Arc::new(OpaEngine::from_proto(&proto)?)); - let policy = SandboxPolicy::try_from(proto.clone())?; - info!("Loaded sidecar policy from control socket bootstrap"); - Ok(( - policy, - opa_engine, - Some(proto), - LoadedPolicyOrigin::Gateway { revision: None }, - )) -} - -fn spawn_sidecar_control_update_watcher( - mut updates: tokio::sync::mpsc::UnboundedReceiver, - provider_credentials: ProviderCredentialState, - agent_proposals: AgentProposals, -) -> tokio::task::JoinHandle<()> { - tokio::spawn(async move { - while let Some(update) = updates.recv().await { - match update { - sidecar_control::ControlUpdate::ProviderEnv { - revision, - provider_child_env, - } => { - if revision <= provider_credentials.snapshot().revision { - continue; - } - let env_count = provider_credentials - .install_child_env_snapshot(revision, provider_child_env); - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("provider_env_revision", serde_json::json!(revision)) - .message(format!( - "Sidecar provider environment refreshed [revision:{revision} env_count:{env_count}]" - )) - .build() - ); - } - sidecar_control::ControlUpdate::Policy { - policy_proto, - policy_hash, - config_revision, - } => { - debug!( - version = policy_proto.version, - policy_hash, - config_revision, - "Received sidecar policy update for process supervisor" - ); - } - sidecar_control::ControlUpdate::AgentProposals { - enabled, - config_revision, - } => { - apply_agent_proposals_enabled( - &agent_proposals, - enabled, - "sidecar control", - Some(config_revision), - None, - skills::install_static_skills, - ); - } - } - } - }) -} - -#[cfg(target_os = "linux")] -fn spawn_sidecar_entrypoint_handler( - mut entrypoint_rx: tokio::sync::mpsc::Receiver, - entrypoint_pid: Arc, - opa_engine: Option>, - retained_proto: Option, - openshell_endpoint: Option, - sandbox_id: Option, - trusted_ssh_socket_path: std::path::PathBuf, -) { - tokio::spawn(async move { - let mut session_started = false; - let mut trusted_supervisor_pid = None; - let terminating = Arc::new(AtomicBool::new(false)); - while let Some(started) = entrypoint_rx.recv().await { - entrypoint_pid.store(started.pid, Ordering::Release); - if started.start_session { - info!( - pid = started.pid, - ssh_socket = %trusted_ssh_socket_path.display(), - "Sidecar process supervisor reported entrypoint start" - ); - } else { - trusted_supervisor_pid = Some(started.pid); - info!( - pid = started.pid, - "Sidecar process supervisor reported initial process anchor" - ); - } - - if let (Some(engine), Some(proto)) = (opa_engine.as_ref(), retained_proto.as_ref()) { - match engine.reload_from_proto_with_pid(proto, started.pid) { - Ok(()) => info!( - pid = started.pid, - "Policy binary symlink resolution complete for sidecar process anchor" - ), - Err(err) => warn!( - error = %err, - pid = started.pid, - "Failed to rebuild OPA engine with sidecar process anchor PID" - ), - } - } - - if started.start_session - && !session_started - && let (Some(endpoint), Some(id)) = - (openshell_endpoint.as_ref(), sandbox_id.as_ref()) - { - let Some(supervisor_pid) = trusted_supervisor_pid else { - warn!( - pid = started.pid, - "Ignoring sidecar entrypoint event before authenticated supervisor anchor" - ); - continue; - }; - openshell_supervisor_process::supervisor_session::spawn( - endpoint.clone(), - id.clone(), - trusted_ssh_socket_path.clone(), - None, - Some(supervisor_pid), - Arc::clone(&terminating), - ); - session_started = true; - info!("sidecar supervisor session task spawned"); - } - } - terminating.store(true, Ordering::Release); - }); -} - -fn sidecar_ca_file_paths() -> Option<(std::path::PathBuf, std::path::PathBuf)> { - let tls_dir = std::env::var(openshell_core::sandbox_env::PROXY_TLS_DIR) - .unwrap_or_else(|_| SIDECAR_TLS_DIR.to_string()); - let cert = std::path::Path::new(&tls_dir).join(SIDECAR_CA_CERT); - let bundle = std::path::Path::new(&tls_dir).join(SIDECAR_CA_BUNDLE); - (cert.exists() && bundle.exists()).then_some((cert, bundle)) -} - -fn process_policy_for_topology( - policy: &SandboxPolicy, - sidecar_network_enforcement: bool, -) -> Result { - let mut process_policy = policy.clone(); - if sidecar_network_enforcement && matches!(process_policy.network.mode, NetworkMode::Proxy) { - let proxy = process_policy - .network - .proxy - .get_or_insert(ProxyPolicy { http_addr: None }); - if proxy.http_addr.is_none() { - proxy.http_addr = Some(SIDECAR_PROCESS_PROXY_ADDR.parse().into_diagnostic()?); - } - } - Ok(process_policy) -} - -/// Flush aggregated denial summaries to the gateway via `SubmitPolicyAnalysis`. -async fn flush_proposals_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summaries: Vec, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialSummary, L7RequestSample}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summaries: Vec = summaries - .into_iter() - .map(|s| DenialSummary { - sandbox_id: String::new(), - host: s.host, - port: u32::from(s.port), - binary: s.binary, - ancestors: s.ancestors, - deny_reason: s.deny_reason, - first_seen_ms: s.first_seen_ms, - last_seen_ms: s.last_seen_ms, - count: s.count, - suppressed_count: 0, - total_count: s.count, - sample_cmdlines: s.sample_cmdlines, - binary_sha256: String::new(), - persistent: false, - denial_stage: s.denial_stage, - l7_request_samples: s - .l7_samples - .into_iter() - .map(|l| L7RequestSample { - method: l.method, - path: l.path, - decision: "deny".to_string(), - count: l.count, - }) - .collect(), - l7_inspection_active: false, - }) - .collect(); - - // Run the mechanistic mapper sandbox-side to generate proposals. - // The gateway is a thin persistence + validation layer — it never - // generates proposals itself. - let proposals = mechanistic_mapper::generate_proposals(&proto_summaries); - - info!( - sandbox_name = %sandbox_name, - summaries = proto_summaries.len(), - proposals = proposals.len(), - "Flushed denial analysis to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - proto_summaries, - proposals, - Vec::new(), - "mechanistic", - ) - .await?; - - Ok(()) -} - -/// Flush an anonymous activity summary to the gateway via `SubmitPolicyAnalysis`. -async fn flush_activity_to_gateway( - endpoint: &str, - sandbox_name: &str, - workspace: &str, - summary: activity_aggregator::FlushableActivitySummary, -) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::{DenialGroupCount, NetworkActivitySummary}; - - let client = CachedOpenShellClient::connect(endpoint).await?; - client.set_workspace(workspace.to_string()); - - let proto_summary = NetworkActivitySummary { - network_activity_count: summary.network_activity_count, - denied_action_count: summary.denied_action_count, - denials_by_group: summary - .denials_by_group - .into_iter() - .map(|(group, count)| DenialGroupCount { - deny_group: group, - denied_count: count, - }) - .collect(), - }; - - info!( - sandbox_name = %sandbox_name, - network_activity_count = proto_summary.network_activity_count, - denied_action_count = proto_summary.denied_action_count, - "Flushed activity summary to gateway" - ); - - client - .submit_policy_analysis( - sandbox_name, - Vec::new(), - Vec::new(), - vec![proto_summary], - "activity", - ) - .await?; - - Ok(()) -} - -// ============================================================================ -// Baseline filesystem path enrichment -// ============================================================================ - -/// Minimum read-only paths required for a proxy-mode sandbox child process to -/// function: dynamic linker, shared libraries, DNS resolution, CA certs, -/// Python venv, openshell logs, process info, and random bytes. -/// -/// `/proc` and `/dev/urandom` are included here for the same reasons they -/// appear in `restrictive_default_policy()`: virtually every process needs -/// them. Before the Landlock per-path fix (#677) these were effectively free -/// because a missing path silently disabled the entire ruleset; now they must -/// be explicit. -const PROXY_BASELINE_READ_ONLY: &[&str] = &[ - "/usr", - "/lib", - "/etc", - "/app", - "/var/log", - "/proc", - "/dev/urandom", -]; - -/// Minimum read-write paths required for a proxy-mode sandbox child process: -/// user working directory and temporary files. -const PROXY_BASELINE_READ_WRITE: &[&str] = &["/sandbox", "/tmp"]; - -/// GPU read-only paths. -/// -/// `/run/nvidia-persistenced`: NVML tries to connect to the persistenced -/// socket at init time. If the directory exists but Landlock denies traversal -/// (EACCES vs ECONNREFUSED), NVML returns `NVML_ERROR_INSUFFICIENT_PERMISSIONS` -/// even though the daemon is optional. Only read/traversal access is needed. -/// -/// `/usr/lib/wsl`: On WSL2, CDI bind-mounts GPU libraries (libdxcore.so, -/// libcuda.so.1.1, etc.) into paths under `/usr/lib/wsl/`. Although `/usr` -/// is already in `PROXY_BASELINE_READ_ONLY`, individual file bind-mounts may -/// not be covered by the parent-directory Landlock rule when the mount crosses -/// a filesystem boundary. Listing `/usr/lib/wsl` explicitly ensures traversal -/// is permitted regardless of Landlock's cross-mount behaviour. -const GPU_BASELINE_READ_ONLY: &[&str] = &[ - "/run/nvidia-persistenced", - "/usr/lib/wsl", // WSL2: CDI-injected GPU library directory -]; - -/// GPU read-write paths (static). -/// -/// `/dev/nvidiactl`, `/dev/nvidia-uvm`, `/dev/nvidia-uvm-tools`, -/// `/dev/nvidia-modeset`: control and UVM devices injected by CDI on native -/// Linux. Landlock restricts `open(2)` on device files even when DAC allows -/// it; these need read-write because NVML/CUDA opens them with `O_RDWR`. -/// These devices do not exist on WSL2 and will be skipped by the existence -/// check in `enrich_proto_baseline_paths()`. -/// -/// `/dev/dxg`: On WSL2, NVIDIA GPUs are exposed through the DXG kernel driver -/// (DirectX Graphics) rather than the native nvidia* devices. CDI injects -/// `/dev/dxg` as the sole GPU device node; it does not exist on native Linux -/// and will be skipped there by the existence check. -/// -/// `/proc`: CUDA writes to `/proc//task//comm` during `cuInit()` -/// to set thread names. Without write access, `cuInit()` returns error 304. -/// Must use `/proc` (not `/proc/self/task`) because Landlock rules bind to -/// inodes and child processes have different procfs inodes than the parent. -/// -/// Per-GPU device files (`/dev/nvidia0`, …) are enumerated at runtime by -/// `enumerate_gpu_device_nodes()` since the count varies. -const GPU_BASELINE_READ_WRITE: &[&str] = &[ - "/dev/nvidiactl", - "/dev/nvidia-uvm", - "/dev/nvidia-uvm-tools", - "/dev/nvidia-modeset", - "/dev/dxg", // WSL2: DXG device (GPU via DirectX kernel driver, injected by CDI) - "/proc", -]; - -/// Returns true if GPU devices are present in the container. -/// -/// Checks both the native Linux NVIDIA control device (`/dev/nvidiactl`) and -/// the WSL2 DXG device (`/dev/dxg`). CDI injects exactly one of these -/// depending on the host kernel; the other will not exist. -fn has_gpu_devices() -> bool { - std::path::Path::new("/dev/nvidiactl").exists() || std::path::Path::new("/dev/dxg").exists() -} - -/// Enumerate per-GPU device nodes (`/dev/nvidia0`, `/dev/nvidia1`, …). -fn enumerate_gpu_device_nodes() -> Vec { - let mut paths = Vec::new(); - if let Ok(entries) = std::fs::read_dir("/dev") { - for entry in entries.flatten() { - let name = entry.file_name(); - let name = name.to_string_lossy(); - if let Some(suffix) = name.strip_prefix("nvidia") { - if suffix.is_empty() || !suffix.chars().all(|c| c.is_ascii_digit()) { - continue; - } - paths.push(entry.path().to_string_lossy().into_owned()); - } - } - } - paths -} - -fn push_unique(paths: &mut Vec, path: String) { - if !paths.iter().any(|p| p == &path) { - paths.push(path); - } -} - -fn collect_baseline_enrichment_paths( - include_proxy: bool, - include_gpu: bool, - gpu_device_nodes: Vec, -) -> (Vec, Vec) { - let mut ro = Vec::new(); - let mut rw = Vec::new(); - - if include_proxy { - for &path in PROXY_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in PROXY_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - } - - if include_gpu { - for &path in GPU_BASELINE_READ_ONLY { - push_unique(&mut ro, path.to_string()); - } - for &path in GPU_BASELINE_READ_WRITE { - push_unique(&mut rw, path.to_string()); - } - for path in gpu_device_nodes { - push_unique(&mut rw, path); - } - } - - // A path promoted to read_write (e.g. /proc for GPU) should not also - // appear in read_only — Landlock handles the overlap correctly but the - // duplicate is confusing when inspecting the effective policy. - ro.retain(|p| !rw.contains(p)); - - (ro, rw) -} - -fn active_baseline_enrichment_paths(include_proxy: bool) -> (Vec, Vec) { - let include_gpu = has_gpu_devices(); - let gpu_device_nodes = if include_gpu { - enumerate_gpu_device_nodes() - } else { - Vec::new() - }; - collect_baseline_enrichment_paths(include_proxy, include_gpu, gpu_device_nodes) -} - -/// Collect all active baseline paths for tests and diagnostics. -/// Returns `(read_only, read_write)` as owned `String` vecs. -#[cfg(test)] -fn baseline_enrichment_paths() -> (Vec, Vec) { - active_baseline_enrichment_paths(true) -} - -fn enrich_proto_baseline_paths_with( - proto: &mut openshell_core::proto::SandboxPolicy, - ro: &[String], - rw: &[String], - path_exists: F, -) -> bool -where - F: Fn(&str) -> bool, -{ - if ro.is_empty() && rw.is_empty() { - return false; - } - - let fs = proto - .filesystem - .get_or_insert_with(|| openshell_core::proto::FilesystemPolicy { - include_workdir: true, - ..Default::default() - }); - - let mut modified = false; - for path in ro { - if !fs.read_only.iter().any(|p| p == path) && !fs.read_write.iter().any(|p| p == path) { - if !path_exists(path) { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - fs.read_only.push(path.clone()); - modified = true; - } - } - for path in rw { - if fs.read_write.iter().any(|p| p == path) { - continue; - } - if !path_exists(path) { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - if fs.read_only.iter().any(|p| p == path) { - if path == "/proc" { - info!( - path, - "Promoting /proc from read-only to read-write for GPU runtime compatibility" - ); - fs.read_only.retain(|p| p != path); - fs.read_write.push(path.clone()); - modified = true; - } - continue; - } - fs.read_write.push(path.clone()); - modified = true; - } - - modified -} - -/// Ensure a proto `SandboxPolicy` includes the baseline filesystem paths -/// required by proxy-mode sandboxes and GPU runtimes. Paths are only added if -/// missing; user-specified paths are never removed. -/// -/// Returns `true` if the policy was modified (caller may want to sync back). -fn enrich_proto_baseline_paths(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - let (ro, rw) = active_baseline_enrichment_paths(!proto.network_policies.is_empty()); - - // Baseline paths are system-injected, not user-specified. Skip paths - // that do not exist in this container image to avoid noisy warnings from - // Landlock and, more critically, to prevent a single missing baseline - // path from abandoning the entire Landlock ruleset under best-effort - // mode (see issue #664). - let modified = enrich_proto_baseline_paths_with(proto, &ro, &rw, |path| { - std::path::Path::new(path).exists() - }); - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } - - modified -} - -fn strip_proto_provider_policy_entries(proto: &mut openshell_core::proto::SandboxPolicy) -> bool { - openshell_policy::strip_provider_rule_names(proto) -} - -fn proto_sync_payload_for_enriched_policy( - proto: &openshell_core::proto::SandboxPolicy, - enriched: bool, -) -> Option { - if !enriched { - return None; - } - - let mut sync_policy = proto.clone(); - strip_proto_provider_policy_entries(&mut sync_policy); - Some(sync_policy) -} - -/// Ensure a `SandboxPolicy` (Rust type) includes the baseline filesystem -/// paths required by proxy-mode sandboxes and GPU runtimes. Used for the -/// local-file code path where no proto is available. -fn enrich_sandbox_baseline_paths(policy: &mut SandboxPolicy) { - let (ro, rw) = - active_baseline_enrichment_paths(matches!(policy.network.mode, NetworkMode::Proxy)); - if ro.is_empty() && rw.is_empty() { - return; - } - - let mut modified = false; - for path in &ro { - let p = std::path::PathBuf::from(path); - if !policy.filesystem.read_only.contains(&p) && !policy.filesystem.read_write.contains(&p) { - if !p.exists() { - debug!( - path, - "Baseline read-only path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_only.push(p); - modified = true; - } - } - for path in &rw { - let p = std::path::PathBuf::from(path); - if policy.filesystem.read_only.contains(&p) || policy.filesystem.read_write.contains(&p) { - continue; - } - if !p.exists() { - debug!( - path, - "Baseline read-write path does not exist, skipping enrichment" - ); - continue; - } - policy.filesystem.read_write.push(p); - modified = true; - } - - if modified { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enriched") - .message("Enriched policy with baseline filesystem paths for proxy mode") - .build() - ); - } -} - -#[cfg(test)] -#[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." -)] -mod baseline_tests { - use super::*; - use openshell_core::policy::{FilesystemPolicy, LandlockPolicy, ProcessPolicy}; - - #[test] - fn proc_not_in_both_read_only_and_read_write_when_gpu_present() { - // When GPU devices are present, /proc is promoted to read_write - // (CUDA needs to write /proc//task//comm). It should - // NOT also appear in read_only. - if !has_gpu_devices() { - // Can't test GPU dedup without GPU devices; skip silently. - return; - } - let (ro, rw) = baseline_enrichment_paths(); - assert!( - rw.contains(&"/proc".to_string()), - "/proc should be in read_write when GPU is present" - ); - assert!( - !ro.contains(&"/proc".to_string()), - "/proc should NOT be in read_only when it is already in read_write" - ); - } - - #[test] - fn proc_in_read_only_without_gpu() { - if has_gpu_devices() { - // On a GPU host we can't test the non-GPU path; skip silently. - return; - } - let (ro, _rw) = baseline_enrichment_paths(); - assert!( - ro.contains(&"/proc".to_string()), - "/proc should be in read_only when GPU is not present" - ); - } - - #[test] - fn baseline_read_write_always_includes_sandbox_and_tmp() { - let (_ro, rw) = baseline_enrichment_paths(); - assert!(rw.contains(&"/sandbox".to_string())); - assert!(rw.contains(&"/tmp".to_string())); - } - - #[test] - fn enumerate_gpu_device_nodes_skips_bare_nvidia() { - // "nvidia" (without a trailing digit) is a valid /dev entry on some - // systems but is not a per-GPU device node. The enumerator must - // not match it. - let nodes = enumerate_gpu_device_nodes(); - assert!( - !nodes.contains(&"/dev/nvidia".to_string()), - "bare /dev/nvidia should not be enumerated: {nodes:?}" - ); - } - - #[test] - fn no_duplicate_paths_in_baseline() { - let (ro, rw) = baseline_enrichment_paths(); - // No path should appear in both lists. - for path in &ro { - assert!( - !rw.contains(path), - "path {path} appears in both read_only and read_write" - ); - } - } - - #[test] - fn proto_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.filesystem = Some(openshell_core::proto::FilesystemPolicy { - read_only: vec!["/tmp".to_string()], - read_write: vec![], - include_workdir: false, - }); - policy.network_policies.insert( - "test".into(), - openshell_core::proto::NetworkPolicyRule { - name: "test-rule".into(), - endpoints: vec![openshell_core::proto::NetworkEndpoint { - host: "example.com".into(), - port: 443, - ..Default::default() - }], - ..Default::default() - }, - ); - - enrich_proto_baseline_paths(&mut policy); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - filesystem.read_only.contains(&"/tmp".to_string()), - "explicit read_only baseline path should be preserved" - ); - assert!( - !filesystem.read_write.contains(&"/tmp".to_string()), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn proto_strip_provider_policy_entries_removes_only_reserved_entries() { - let mut policy = openshell_policy::restrictive_default_policy(); - policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - assert!(strip_proto_provider_policy_entries(&mut policy)); - assert!( - !policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(policy.network_policies.contains_key("sandbox_only")); - assert!(!strip_proto_provider_policy_entries(&mut policy)); - } - - #[test] - fn proto_sync_payload_not_created_for_provider_entries_without_enrichment() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - - assert!(proto_sync_payload_for_enriched_policy(&runtime_policy, false).is_none()); - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "provider-derived rules alone must not trigger sync or mutate runtime policy" - ); - } - - #[test] - fn proto_sync_payload_for_enrichment_strips_provider_entries_without_mutating_runtime_policy() { - let mut runtime_policy = openshell_policy::restrictive_default_policy(); - runtime_policy.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "_provider_work_github".to_string(), - ..Default::default() - }, - ); - runtime_policy.network_policies.insert( - "sandbox_only".to_string(), - openshell_core::proto::NetworkPolicyRule { - name: "sandbox_only".to_string(), - ..Default::default() - }, - ); - - let sync_policy = proto_sync_payload_for_enriched_policy(&runtime_policy, true) - .expect("enrichment should create a sync payload"); - - assert!( - runtime_policy - .network_policies - .contains_key("_provider_work_github"), - "runtime policy must retain provider-derived rules for OPA input" - ); - assert!( - !sync_policy - .network_policies - .contains_key("_provider_work_github") - ); - assert!(sync_policy.network_policies.contains_key("sandbox_only")); - } - - #[test] - fn proto_gpu_enrichment_promotes_proc_without_network_policy() { - let mut policy = openshell_policy::restrictive_default_policy(); - assert!( - policy.network_policies.is_empty(), - "regression setup must exercise the no-network default path" - ); - let (ro, rw) = - collect_baseline_enrichment_paths(false, true, vec!["/dev/nvidia0".to_string()]); - - let enriched = enrich_proto_baseline_paths_with(&mut policy, &ro, &rw, |path| { - matches!(path, "/proc" | "/dev/nvidia0") - }); - - let filesystem = policy.filesystem.expect("filesystem policy"); - assert!( - enriched, - "GPU enrichment should not require network policies" - ); - assert!( - filesystem.read_write.contains(&"/dev/nvidia0".to_string()), - "GPU enrichment should add enumerated device nodes without network policies" - ); - assert!( - !filesystem.read_only.contains(&"/proc".to_string()), - "GPU enrichment should remove /proc from read_only" - ); - assert!( - filesystem.read_write.contains(&"/proc".to_string()), - "GPU enrichment should promote /proc to read_write" - ); - } - - #[test] - fn gpu_baseline_read_write_contains_dxg() { - // /dev/dxg must be present so WSL2 sandboxes get the Landlock - // read-write rule for the CDI-injected DXG device. The existence - // check in enrich_proto_baseline_paths() skips it on native Linux. - assert!( - GPU_BASELINE_READ_WRITE.contains(&"/dev/dxg"), - "/dev/dxg must be in GPU_BASELINE_READ_WRITE for WSL2 support" - ); - } - - #[test] - fn local_enrichment_preserves_explicit_read_only_for_baseline_read_write_paths() { - let mut policy = SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy { - read_only: vec![std::path::PathBuf::from("/tmp")], - read_write: vec![], - include_workdir: false, - }, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - }; - - enrich_sandbox_baseline_paths(&mut policy); - - assert!( - policy - .filesystem - .read_only - .contains(&std::path::PathBuf::from("/tmp")), - "explicit read_only baseline path should be preserved" - ); - assert!( - !policy - .filesystem - .read_write - .contains(&std::path::PathBuf::from("/tmp")), - "baseline enrichment must not promote explicit read_only /tmp to read_write" - ); - } - - #[test] - fn gpu_baseline_read_only_contains_usr_lib_wsl() { - // /usr/lib/wsl must be present so CDI-injected WSL2 GPU library - // bind-mounts are accessible under Landlock. Skipped on native Linux. - assert!( - GPU_BASELINE_READ_ONLY.contains(&"/usr/lib/wsl"), - "/usr/lib/wsl must be in GPU_BASELINE_READ_ONLY for WSL2 CDI library paths" - ); - } - - #[test] - fn has_gpu_devices_reflects_dxg_or_nvidiactl() { - // Verify the OR logic: result must match the manual disjunction of - // the two path checks. Passes in all environments. - let nvidiactl = std::path::Path::new("/dev/nvidiactl").exists(); - let dxg = std::path::Path::new("/dev/dxg").exists(); - assert_eq!( - has_gpu_devices(), - nvidiactl || dxg, - "has_gpu_devices() should be true iff /dev/nvidiactl or /dev/dxg exists" - ); - } -} - -/// Returns `true` if the error is transient and worth retrying. -/// -/// Walks the `miette::Report` error chain looking for a `tonic::Status`. If -/// found, only the gRPC codes that represent transient failures are retryable. -/// If no `tonic::Status` is present (e.g. a raw connection error), assume the -/// failure is transient. -fn is_retryable_error(err: &miette::Report) -> bool { - let mut source: Option<&dyn std::error::Error> = Some(err.as_ref()); - while let Some(e) = source { - if let Some(status) = e.downcast_ref::() { - return matches!( - status.code(), - tonic::Code::Unavailable - | tonic::Code::DeadlineExceeded - | tonic::Code::ResourceExhausted - | tonic::Code::Aborted - | tonic::Code::Internal - | tonic::Code::Unknown - ); - } - source = e.source(); - } - true -} - -/// Retry a gRPC operation with exponential backoff (capped at 4 s). -/// -/// Non-transient gRPC errors (e.g. `NOT_FOUND`, `INVALID_ARGUMENT`, -/// `PERMISSION_DENIED`) are returned immediately without retrying. -async fn grpc_retry(op_name: &str, f: F) -> Result -where - F: Fn() -> Fut, - Fut: Future>, -{ - let mut last_err = None; - for attempt in 1..=5u32 { - match f().await { - Ok(val) => return Ok(val), - Err(e) => { - if !is_retryable_error(&e) { - return Err(e); - } - if attempt < 5 { - warn!( - attempt, - max_attempts = 5, - error = %e, - "{op_name} failed, retrying" - ); - let backoff = Duration::from_secs((1u64 << (attempt - 1)).min(4)); - tokio::time::sleep(backoff).await; - } - last_err = Some(e); - } - } - } - Err(miette::miette!( - "{op_name} failed after 5 attempts: {}", - last_err.expect("loop executed at least once") - )) -} - -/// Load sandbox policy from local files or gRPC. -/// -/// Priority: -/// 1. If `policy_rules` and `policy_data` are provided, load OPA engine from local files -/// 2. If `sandbox_id` and `openshell_endpoint` are provided, fetch via gRPC -/// 3. If the server returns no policy, discover from disk or use restrictive default -/// 4. Otherwise, return an error -/// -/// Returns the policy, the OPA engine, and (for gRPC mode) the original proto -/// policy. The proto is retained so the OPA engine can be rebuilt with symlink -/// resolution after the container entrypoint starts. -async fn load_policy( - sandbox_id: Option, - sandbox: Option, - openshell_endpoint: Option, - policy_rules: Option, - policy_data: Option, -) -> Result<( - SandboxPolicy, - Option>, - Option, - MiddlewareRegistryStatus, - LoadedPolicyOrigin, - bool, -)> { - // File mode: load OPA engine from rego rules + YAML data (dev override) - if let (Some(policy_file), Some(data_file)) = (&policy_rules, &policy_data) { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "loading") - .unmapped("policy_rules", serde_json::json!(policy_file)) - .unmapped("policy_data", serde_json::json!(data_file)) - .message(format!( - "Loading OPA policy engine from local files [rules:{policy_file} data:{data_file}]" - )) - .build()); - let validate_middleware_config = |implementation: &str, config: &prost_types::Struct| { - openshell_supervisor_middleware_builtins::validate_config(implementation, config) - .map_err(|error| error.to_string()) - }; - let engine = OpaEngine::from_files_with_middleware_config( - std::path::Path::new(policy_file), - std::path::Path::new(data_file), - Some(&validate_middleware_config), - )?; - let middleware_registry = - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - engine.replace_middleware_registry(middleware_registry)?; - let config = engine.query_sandbox_config()?; - let mut policy = SandboxPolicy { - version: 1, - filesystem: config.filesystem, - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr: None }), - }, - landlock: config.landlock, - process: config.process, - }; - enrich_sandbox_baseline_paths(&mut policy); - // File mode has no operator-registered middleware to connect. - return Ok(( - policy, - Some(Arc::new(engine)), - None, - MiddlewareRegistryStatus::Synchronized, - LoadedPolicyOrigin::LocalOverride, - false, - )); - } - - // gRPC mode: fetch typed proto policy, construct OPA engine from baked rules + proto data - if let (Some(id), Some(endpoint)) = (&sandbox_id, &openshell_endpoint) { - info!( - sandbox_id = %id, - endpoint = %endpoint, - "Fetching sandbox policy via gRPC" - ); - let mut snapshot = grpc_retry("Policy fetch", || { - openshell_core::grpc_client::fetch_settings_snapshot(endpoint, id) - }) - .await?; - - let mut proto_policy = if let Some(p) = snapshot.policy.clone() { - p - } else { - // No policy configured on the server. Discover from disk or - // fall back to the restrictive default, then sync to the - // gateway so it becomes the authoritative baseline. - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "discovery") - .message("Server returned no policy; attempting local discovery") - .build() - ); - let mut discovered = discover_policy_from_disk_or_default(); - // Enrich before syncing so the gateway baseline includes - // baseline paths from the start. - enrich_proto_baseline_paths(&mut discovered); - strip_proto_provider_policy_entries(&mut discovered); - let sandbox = sandbox.as_deref().ok_or_else(|| { - miette::miette!( - "Cannot sync discovered policy: sandbox not available.\n\ - Set OPENSHELL_SANDBOX or --sandbox to enable policy sync." - ) - })?; - - // Sync and re-fetch over a single connection to avoid extra - // TLS handshakes. - let ws = snapshot.workspace.clone(); - snapshot = grpc_retry("Policy discovery sync", || { - openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox, - &discovered, - &ws, - ) - }) - .await?; - snapshot.policy.clone().ok_or_else(|| { - miette::miette!("Server still returned no policy after sync — this is a bug") - })? - }; - - // True only while `snapshot` describes the exact policy that will be - // constructed below. If enrichment cannot be synced and re-fetched, - // the policy remains enforceable but cannot be acknowledged by - // inferred structural equality. - let mut policy_bound_to_snapshot = true; - - // Ensure baseline filesystem paths are present for proxy-mode - // sandboxes. If the policy was enriched, sync the updated version - // back to the gateway so users can see the effective policy. - let enriched = enrich_proto_baseline_paths(&mut proto_policy); - let sync_policy = proto_sync_payload_for_enriched_policy(&proto_policy, enriched); - if let Some(sync_policy) = sync_policy { - if let Some(sandbox_name) = sandbox.as_deref() { - match openshell_core::grpc_client::sync_policy_and_fetch_snapshot( - endpoint, - id, - sandbox_name, - &sync_policy, - &snapshot.workspace, - ) - .await - { - Ok(canonical) => { - if let Some(policy) = canonical.policy.clone() { - proto_policy = policy; - snapshot = canonical; - } else { - policy_bound_to_snapshot = false; - warn!( - "Gateway returned no policy after enrichment sync; initial revision will be reconciled" - ); - } - } - Err(e) => { - policy_bound_to_snapshot = false; - warn!( - error = %e, - "Failed to sync enriched policy back to gateway; initial revision will be reconciled" - ); - } - } - } else { - policy_bound_to_snapshot = false; - } - } - - let loaded_policy_revision = - policy_bound_to_snapshot.then(|| LoadedPolicyRevision::from_snapshot(&snapshot)); - - // Build OPA engine from baked-in rules + typed proto data. - // In cluster mode, proxy networking is always enabled so OPA is - // always required for allow/deny decisions. - // The initial load uses pid=0 (no symlink resolution) because the - // container hasn't started yet. After the entrypoint spawns, the - // engine is rebuilt with the real PID for symlink resolution. - info!("Creating OPA engine from proto policy data"); - let engine = match OpaEngine::from_proto(&proto_policy) { - Ok(engine) => engine, - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - return Err(e); - } - }; - - // Install the in-process catalog before any external connection can - // fail. A newly started sandbox must always be able to resolve built-in - // bindings, even while operator-run services are unavailable. - install_builtin_middleware_registry(&engine).await?; - - // Connect operator-registered middleware services. A connect/describe - // failure keeps the built-in registry active so each request's - // `on_error` policy governs matched traffic. The policy poll loop - // retries the install without waiting for a config change. - let middleware_services = snapshot.supervisor_middleware_services.clone(); - let middleware_registry_status = if middleware_services.is_empty() { - MiddlewareRegistryStatus::Synchronized - } else if let Err(error) = grpc_retry("Middleware connect", || { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - middleware_services.clone(), - ) - }) - .await - .and_then(|registry| engine.replace_middleware_registry(registry)) - { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "degraded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(middleware_services.len()) - ) - .message(format!( - "Supervisor middleware connect failed at startup; continuing with built-in middleware only, per-request on_error governs matched requests [error:{error}]" - )) - .build() - ); - MiddlewareRegistryStatus::NeedsReconciliation - } else { - MiddlewareRegistryStatus::Synchronized - }; - let opa_engine = Some(Arc::new(engine)); - - let policy = match SandboxPolicy::try_from(proto_policy.clone()) { - Ok(policy) => policy, - Err(e) => { - report_initial_policy_failure(endpoint, id, loaded_policy_revision.as_ref(), &e) - .await; - return Err(e); - } - }; - return Ok(( - policy, - opa_engine, - Some(proto_policy), - middleware_registry_status, - LoadedPolicyOrigin::Gateway { - revision: loaded_policy_revision, - }, - agent_proposals_enabled_from_settings(&snapshot.settings), - )); - } - - // No policy source available - Err(miette::miette!( - "Sandbox policy required. Provide one of:\n\ - - --policy-rules and --policy-data (or OPENSHELL_POLICY_RULES and OPENSHELL_POLICY_DATA env vars)\n\ - - --sandbox-id and --openshell-endpoint (or OPENSHELL_SANDBOX_ID and OPENSHELL_ENDPOINT env vars)" - )) -} - -/// Try to discover a sandbox policy from the well-known disk path, falling -/// back to the legacy path, then to the hardcoded restrictive default. -fn discover_policy_from_disk_or_default() -> openshell_core::proto::SandboxPolicy { - let primary = std::path::Path::new(openshell_policy::CONTAINER_POLICY_PATH); - if primary.exists() { - return discover_policy_from_path(primary); - } - let legacy = std::path::Path::new(openshell_policy::LEGACY_CONTAINER_POLICY_PATH); - if legacy.exists() { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "legacy_path", - serde_json::json!(legacy.display().to_string()) - ) - .unmapped("new_path", serde_json::json!(primary.display().to_string())) - .message(format!( - "Policy found at legacy path; consider moving [legacy_path:{} new_path:{}]", - legacy.display(), - primary.display() - )) - .build() - ); - return discover_policy_from_path(legacy); - } - discover_policy_from_path(primary) -} - -/// Try to read a sandbox policy YAML from `path`, falling back to the -/// hardcoded restrictive default if the file is missing or invalid. -fn discover_policy_from_path(path: &std::path::Path) -> openshell_core::proto::SandboxPolicy { - use openshell_policy::{ - parse_sandbox_policy, restrictive_default_policy, validate_sandbox_policy, - }; - - let Ok(yaml) = std::fs::read_to_string(path) else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "default") - .message(format!( - "No policy file on disk, using restrictive default [path:{}]", - path.display() - )) - .build() - ); - return restrictive_default_policy(); - }; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .message(format!( - "Loaded sandbox policy from container disk [path:{}]", - path.display() - )) - .build() - ); - match parse_sandbox_policy(&yaml) { - Ok(policy) => { - // Validate the disk-loaded policy for safety. - if let Err(violations) = validate_sandbox_policy(&policy) { - let messages: Vec = violations.iter().map(ToString::to_string).collect(); - ocsf_emit!(DetectionFindingBuilder::new(ocsf_ctx()) - .activity(ActivityId::Open) - .severity(SeverityId::Medium) - .action(ActionId::Denied) - .disposition(DispositionId::Blocked) - .finding_info( - FindingInfo::new( - "unsafe-disk-policy", - "Unsafe Disk Policy Content", - ) - .with_desc(&format!( - "Disk policy at {} contains unsafe content: {}", - path.display(), - messages.join("; "), - )), - ) - .message(format!( - "Disk policy contains unsafe content, using restrictive default [path:{}]", - path.display() - )) - .build()); - return restrictive_default_policy(); - } - policy - } - Err(e) => { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "fallback") - .message(format!( - "Failed to parse disk policy, using restrictive default [path:{} error:{e}]", - path.display() - )) - .build()); - restrictive_default_policy() - } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -enum MiddlewareRegistryStatus { - Synchronized, - NeedsReconciliation, -} - -/// True when the installed middleware registry no longer matches the desired -/// service set and must be rebuilt (reconnecting every delivered service). -/// -/// A policy-only change never requires a rebuild: middleware configs were -/// validated at gateway admission and the installed registry's manifests -/// already cover the unchanged service set, so requiring the services to be -/// reachable would only let a middleware outage block the policy update. -fn middleware_registry_needs_rebuild( - registry_status: MiddlewareRegistryStatus, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], -) -> bool { - registry_status == MiddlewareRegistryStatus::NeedsReconciliation - || current_services != desired_services -} - -fn gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy: bool, - current_policy_hash: &str, - desired_policy_hash: &str, - current_services: &[openshell_core::proto::SupervisorMiddlewareService], - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], - registry_status: MiddlewareRegistryStatus, -) -> bool { - reloads_gateway_policy - && (current_policy_hash != desired_policy_hash - || middleware_registry_needs_rebuild( - registry_status, - current_services, - desired_services, - )) -} - -/// Identity returned with the exact policy snapshot used to construct OPA. -#[derive(Clone, Debug, PartialEq, Eq)] -struct LoadedPolicyRevision { - version: u32, - policy_hash: String, - config_revision: u64, - policy_source: openshell_core::proto::PolicySource, -} - -/// Identifies where the policy currently loaded into OPA came from. -/// -/// A missing gateway revision means the policy was loaded from the gateway but -/// could not be bound to an authoritative snapshot (for example, enrichment -/// sync failed). That state must reconcile on the first successful poll. A -/// local-file override is different: gateway policy revisions are observed for -/// settings/provider refreshes but must never replace the explicit local OPA -/// policy. -#[derive(Clone, Debug, PartialEq, Eq)] -enum LoadedPolicyOrigin { - LocalOverride, - Gateway { - revision: Option, - }, -} - -impl LoadedPolicyOrigin { - fn allows_gateway_policy_reload(&self) -> bool { - matches!(self, Self::Gateway { .. }) - } -} - -impl LoadedPolicyRevision { - fn from_snapshot(snapshot: &openshell_core::grpc_client::SettingsPollResult) -> Self { - Self { - version: snapshot.version, - policy_hash: snapshot.policy_hash.clone(), - config_revision: snapshot.config_revision, - policy_source: snapshot.policy_source, - } - } -} - -/// A sandbox-scoped policy revision that was constructed successfully at -/// startup and must be acknowledged to the gateway exactly once. -#[derive(Clone, Debug, PartialEq, Eq)] -struct InitialPolicyAck { - version: u32, - policy_hash: String, - config_revision: u64, -} - -#[derive(Clone, Debug, PartialEq, Eq)] -struct PolicyStatusUpdate { - version: u32, - loaded: bool, - error: String, - initial_policy_hash: Option, -} - -impl PolicyStatusUpdate { - fn initial_loaded(ack: &InitialPolicyAck) -> Self { - Self { - version: ack.version, - loaded: true, - error: String::new(), - initial_policy_hash: Some(ack.policy_hash.clone()), - } - } - - fn loaded(version: u32) -> Self { - Self { - version, - loaded: true, - error: String::new(), - initial_policy_hash: None, - } - } - - fn failed(version: u32, error: String) -> Self { - Self { - version, - loaded: false, - error, - initial_policy_hash: None, - } - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -enum InitialPollDisposition { - Acknowledge(InitialPolicyAck), - Reconcile, - TrackOnly, -} - -/// Determine whether the initially loaded policy corresponds to an -/// authoritative sandbox-scoped revision that must be acknowledged. -/// -/// Returns `Some` only for sandbox-sourced revisions (version > 0) whose -/// captured gateway identity matches the current version and hash. Global -/// policies, local-file development policies, version zero, and changed -/// identities yield `None`, so those paths never emit a sandbox-revision -/// acknowledgement. -fn initial_policy_ack_candidate( - loaded: Option<&LoadedPolicyRevision>, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> Option { - let loaded = loaded?; - if loaded.policy_source != openshell_core::proto::PolicySource::Sandbox - || canonical.policy_source != openshell_core::proto::PolicySource::Sandbox - { - return None; - } - if loaded.version == 0 || canonical.version == 0 { - return None; - } - if loaded.version != canonical.version - || loaded.policy_hash != canonical.policy_hash - || canonical.config_revision < loaded.config_revision - { - return None; - } - Some(InitialPolicyAck { - version: loaded.version, - policy_hash: loaded.policy_hash.clone(), - config_revision: canonical.config_revision, - }) -} - -fn initial_poll_disposition( - origin: &LoadedPolicyOrigin, - canonical: &openshell_core::grpc_client::SettingsPollResult, -) -> InitialPollDisposition { - match origin { - LoadedPolicyOrigin::LocalOverride => InitialPollDisposition::TrackOnly, - LoadedPolicyOrigin::Gateway { revision } => { - initial_policy_ack_candidate(revision.as_ref(), canonical).map_or( - InitialPollDisposition::Reconcile, - InitialPollDisposition::Acknowledge, - ) - } - } -} - -/// Deliver policy status updates independently from policy reconciliation. -/// -/// The channel is FIFO, so a delayed older status can never arrive after a -/// newer status and move the gateway's active version backward. Delivery uses -/// the existing bounded retry, but failures never delay policy enforcement. -async fn run_policy_status_reporter( - client: openshell_core::grpc_client::CachedOpenShellClient, - sandbox_id: String, - mut updates: tokio::sync::mpsc::UnboundedReceiver, -) { - 'updates: while let Some(update) = updates.recv().await { - let operation = if update.initial_policy_hash.is_some() { - "Initial policy acknowledgement" - } else { - "Policy status report" - }; - let mut attempt = 1_u32; - loop { - let sandbox_id = sandbox_id.clone(); - let error = update.error.clone(); - let client = client.clone(); - match client - .report_policy_status(&sandbox_id, update.version, update.loaded, &error) - .await - { - Ok(()) => break, - Err(error) if is_retryable_error(&error) => { - let backoff = Duration::from_secs(1_u64 << attempt.saturating_sub(1).min(5)); - warn!( - %error, - attempt, - version = update.version, - loaded = update.loaded, - retry_in_secs = backoff.as_secs(), - "{operation} failed transiently; retaining ordered update" - ); - tokio::time::sleep(backoff).await; - attempt = attempt.saturating_add(1); - } - Err(error) => { - warn!( - %error, - version = update.version, - loaded = update.loaded, - "Discarding terminal policy status update" - ); - continue 'updates; - } - } - } - - if let Some(policy_hash) = update.initial_policy_hash { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("version", serde_json::json!(update.version)) - .unmapped("policy_hash", serde_json::json!(policy_hash)) - .message(format!( - "Acknowledged initial policy revision as loaded [version:{}]", - update.version - )) - .build() - ); - } - } -} - -fn enqueue_policy_status(sender: &UnboundedSender, update: PolicyStatusUpdate) { - let version = update.version; - if let Err(error) = sender.send(update) { - warn!( - %error, - version, - "Policy status reporter unavailable during shutdown" - ); - } -} - -/// Best-effort `FAILED` acknowledgement when initial policy construction or -/// conversion fails. -/// -/// Uses the revision identity captured with the policy that failed to build, -/// and preserves the original construction error as the reported message. A -/// delivery failure here is swallowed so it can never mask that error. -async fn report_initial_policy_failure( - endpoint: &str, - sandbox_id: &str, - revision: Option<&LoadedPolicyRevision>, - error: &miette::Report, -) { - let Some(revision) = revision.filter(|revision| { - revision.version > 0 - && revision.policy_source == openshell_core::proto::PolicySource::Sandbox - }) else { - return; - }; - let client = match openshell_core::grpc_client::CachedOpenShellClient::connect(endpoint).await { - Ok(client) => client, - Err(e) => { - warn!(error = %e, "Failed to connect to report initial policy failure"); - return; - } - }; - let message = error.to_string(); - if let Err(e) = grpc_retry("Initial policy failure report", || { - let client = client.clone(); - let message = message.clone(); - async move { - client - .report_policy_status(sandbox_id, revision.version, false, &message) - .await - } - }) - .await - { - warn!(error = %e, version = revision.version, "Failed to report initial policy failure"); - } -} - -/// Background loop that polls the server for policy updates. -/// -/// When a new version is detected, attempts to reload the OPA engine via -/// `reload_from_proto_with_pid()`. Reports load success/failure back to the -/// server. On failure, the previous engine is untouched (LKG behavior). -/// -/// When the entrypoint PID is available, policy reloads include symlink -/// resolution for binary paths via the container filesystem. -struct PolicyPollLoopContext { - endpoint: String, - sandbox_id: String, - opa_engine: Arc, - /// Source of the policy currently loaded into OPA. This distinguishes an - /// explicit local-file override from an unbound gateway revision so the - /// former is never replaced by policy polling. - loaded_policy_origin: LoadedPolicyOrigin, - entrypoint_pid: Arc, - interval_secs: u64, - ocsf_enabled: Arc, - provider_credentials: ProviderCredentialState, - policy_local_ctx: Option>, - agent_proposals: AgentProposals, - middleware_registry_status: MiddlewareRegistryStatus, - sidecar_control_publisher: Option, - workspace_tx: tokio::sync::watch::Sender, -} - -async fn connect_middleware_registry( - services: &[openshell_core::proto::SupervisorMiddlewareService], -) -> Result { - openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - services.to_vec(), - ) - .await -} - -async fn install_builtin_middleware_registry(opa_engine: &OpaEngine) -> Result<()> { - let registry = openshell_supervisor_middleware::MiddlewareRegistry::connect_services( - openshell_supervisor_middleware_builtins::services(), - Vec::new(), - ) - .await?; - opa_engine.replace_middleware_registry(registry) -} - -async fn reconcile_middleware_registry( - opa_engine: &OpaEngine, - desired_services: &[openshell_core::proto::SupervisorMiddlewareService], - current_services: &mut Vec, - status: &mut MiddlewareRegistryStatus, -) { - if *status == MiddlewareRegistryStatus::Synchronized - && desired_services == current_services.as_slice() - { - return; - } - - match connect_middleware_registry(desired_services) - .await - .and_then(|registry| opa_engine.replace_middleware_registry(registry)) - { - Ok(()) => { - current_services.clear(); - current_services.extend_from_slice(desired_services); - *status = MiddlewareRegistryStatus::Synchronized; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(current_services.len()) - ) - .message(format!( - "Supervisor middleware registry reloaded [service_count:{}]", - current_services.len() - )) - .build() - ); - } - Err(error) => { - // Emit only on the transition into the failed state to avoid - // repeating the same finding on every poll during an outage. - if *status == MiddlewareRegistryStatus::Synchronized { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .message(format!( - "Supervisor middleware registry reload failed, keeping last-known-good registry [error:{error}]" - )) - .build() - ); - } - *status = MiddlewareRegistryStatus::NeedsReconciliation; - } - } -} - -async fn run_policy_poll_loop(ctx: PolicyPollLoopContext) -> Result<()> { - use openshell_core::grpc_client::CachedOpenShellClient; - use openshell_core::proto::PolicySource; - use std::sync::atomic::Ordering; - - let client = CachedOpenShellClient::connect(&ctx.endpoint).await?; - let (status_sender, status_receiver) = tokio::sync::mpsc::unbounded_channel(); - tokio::spawn(run_policy_status_reporter( - client.clone(), - ctx.sandbox_id.clone(), - status_receiver, - )); - - let mut current_config_revision: u64 = 0; - let mut current_provider_env_revision: u64 = ctx.provider_credentials.snapshot().revision; - let mut current_policy_hash = String::new(); - let mut current_middleware_services = Vec::new(); - let mut middleware_registry_status = ctx.middleware_registry_status; - let mut current_settings: std::collections::HashMap< - String, - openshell_core::proto::EffectiveSetting, - > = std::collections::HashMap::new(); - let reloads_gateway_policy = ctx.loaded_policy_origin.allows_gateway_policy_reload(); - let mut last_failed_runtime_revision: Option<(u64, String)> = None; - - // A first poll that does not match the policy already loaded into OPA must - // pass through the normal reconciliation path immediately. It must never - // seed the applied-state trackers before OPA actually loads it. - let mut pending_result = None; - - // Initialize revision from the first poll and acknowledge the initial - // policy revision the supervisor actually loaded. A mismatched result is - // reconciled below instead of being recorded as already applied. - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - match initial_poll_disposition(&ctx.loaded_policy_origin, &result) { - InitialPollDisposition::Acknowledge(candidate) => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(candidate.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = candidate.config_revision; - current_policy_hash.clone_from(&candidate.policy_hash); - current_middleware_services = result.supervisor_middleware_services; - current_settings = result.settings; - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::initial_loaded(&candidate), - ); - debug!( - config_revision = current_config_revision, - "Settings poll: initial policy matches loaded revision" - ); - } - InitialPollDisposition::Reconcile => pending_result = Some(result), - InitialPollDisposition::TrackOnly => { - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "initial settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - current_config_revision = result.config_revision; - current_policy_hash = result.policy_hash.clone(); - current_middleware_services = result.supervisor_middleware_services; - current_settings = result.settings; - debug!( - config_revision = current_config_revision, - "Settings poll: tracking gateway config while preserving local policy override" - ); - } - } - } - Err(e) => { - warn!(error = %e, "Settings poll: failed to fetch initial version, will retry"); - } - } - - let interval = Duration::from_secs(ctx.interval_secs); - loop { - let result = if let Some(result) = pending_result.take() { - result - } else { - tokio::time::sleep(interval).await; - match client.poll_settings(&ctx.sandbox_id).await { - Ok(result) => { - let _ = ctx.workspace_tx.send(client.workspace()); - result - } - Err(e) => { - debug!(error = %e, "Settings poll: server unreachable, will retry"); - continue; - } - } - }; - - let config_changed = result.config_revision != current_config_revision; - let provider_env_changed = result.provider_env_revision != current_provider_env_revision; - let policy_changed = result.policy_hash != current_policy_hash; - let middleware_registry_changed = middleware_registry_needs_rebuild( - middleware_registry_status, - ¤t_middleware_services, - &result.supervisor_middleware_services, - ); - let policy_runtime_changed = gateway_policy_runtime_needs_reconciliation( - reloads_gateway_policy, - ¤t_policy_hash, - &result.policy_hash, - ¤t_middleware_services, - &result.supervisor_middleware_services, - middleware_registry_status, - ); - - // A local policy override is not coupled to the gateway policy - // snapshot, so its service registry can still be reconciled alone. - // Gateway policy snapshots, however, must install policy and registry - // as one generation below. - if !reloads_gateway_policy { - reconcile_middleware_registry( - &ctx.opa_engine, - &result.supervisor_middleware_services, - &mut current_middleware_services, - &mut middleware_registry_status, - ) - .await; - } - - if !config_changed && !provider_env_changed && !policy_runtime_changed { - continue; - } - - if config_changed || provider_env_changed { - // Log which settings changed. - log_setting_changes(¤t_settings, &result.settings); - - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Other, "detected") - .unmapped("old_config_revision", serde_json::json!(current_config_revision)) - .unmapped("new_config_revision", serde_json::json!(result.config_revision)) - .unmapped("policy_changed", serde_json::json!(policy_changed)) - .unmapped("provider_env_changed", serde_json::json!(provider_env_changed)) - .message(format!( - "Settings poll: config change detected [old_revision:{current_config_revision} new_revision:{} policy_changed:{policy_changed} provider_env_changed:{provider_env_changed}]", - result.config_revision - )) - .build()); - } - - if provider_env_changed { - match openshell_core::grpc_client::fetch_provider_environment( - &ctx.endpoint, - &ctx.sandbox_id, - ) - .await - { - Ok(env_result) => { - ctx.provider_credentials.install_environment( - env_result.provider_env_revision, - env_result.environment, - env_result.credential_expires_at_ms, - env_result.dynamic_credentials, - ); - let child_env = ctx.provider_credentials.child_env_with_gcp_resolved(); - let env_count = child_env.len(); - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher.publish_provider_env( - env_result.provider_env_revision, - child_env.clone(), - ); - } - current_provider_env_revision = env_result.provider_env_revision; - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "provider_env_revision", - serde_json::json!(env_result.provider_env_revision) - ) - .message(format!( - "Provider environment refreshed [revision:{} env_count:{env_count}]", - env_result.provider_env_revision - )) - .build() - ); - } - Err(e) => { - warn!( - error = %e, - provider_env_revision = result.provider_env_revision, - "Settings poll: failed to refresh provider environment" - ); - } - } - } - - if policy_runtime_changed { - let pid = ctx.entrypoint_pid.load(Ordering::Acquire); - let runtime_result = match result.policy.as_ref() { - Some(policy) if middleware_registry_changed => { - match connect_middleware_registry(&result.supervisor_middleware_services).await - { - Ok(registry) => ctx - .opa_engine - .reload_policy_and_middleware_from_proto_with_pid( - policy, pid, registry, - ), - Err(error) => Err(error), - } - } - // Policy-only change: the installed registry already matches - // the delivered service set, so swap the engine alone. This - // must not require middleware reachability. - Some(policy) => ctx.opa_engine.reload_from_proto_with_pid(policy, pid), - None => Err(miette::miette!( - "runtime reload requires a policy payload but none was returned" - )), - }; - - match runtime_result { - Ok(()) => { - let policy = result - .policy - .as_ref() - .expect("successful runtime reload requires a policy payload"); - if policy_changed { - if let Some(policy_local_ctx) = ctx.policy_local_ctx.as_ref() { - policy_local_ctx.set_current_policy(policy.clone()).await; - } - if let Some(publisher) = ctx.sidecar_control_publisher.as_ref() { - publisher.publish_policy( - policy.clone(), - result.policy_hash.clone(), - result.config_revision, - ); - } - if result.global_policy_version > 0 { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .unmapped("global_version", serde_json::json!(result.global_policy_version)) - .message(format!( - "Policy reloaded successfully (global) [policy_hash:{} global_version:{}]", - result.policy_hash, - result.global_policy_version - )) - .build()); - } else { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped("policy_hash", serde_json::json!(&result.policy_hash)) - .message(format!( - "Policy reloaded successfully [policy_hash:{}]", - result.policy_hash - )) - .build() - ); - } - if result.version > 0 && result.policy_source == PolicySource::Sandbox { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::loaded(result.version), - ); - } - } - - if middleware_registry_changed { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "loaded") - .unmapped( - "supervisor_middleware_service_count", - serde_json::json!(result.supervisor_middleware_services.len()) - ) - .message(format!( - "Supervisor policy runtime reloaded atomically [service_count:{}]", - result.supervisor_middleware_services.len() - )) - .build()); - } - - current_policy_hash.clone_from(&result.policy_hash); - current_middleware_services.clone_from(&result.supervisor_middleware_services); - middleware_registry_status = MiddlewareRegistryStatus::Synchronized; - last_failed_runtime_revision = None; - } - Err(e) => { - let failed_revision = (result.config_revision, result.policy_hash.clone()); - if last_failed_runtime_revision.as_ref() != Some(&failed_revision) { - ocsf_emit!(ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Medium) - .status(StatusId::Failure) - .state(StateId::Other, "failed") - .unmapped("version", serde_json::json!(result.version)) - .unmapped("error", serde_json::json!(e.to_string())) - .message(format!( - "Policy and middleware runtime reload failed, keeping last-known-good runtime [version:{} error:{e}]", - result.version - )) - .build()); - if policy_changed - && result.version > 0 - && result.policy_source == PolicySource::Sandbox - { - enqueue_policy_status( - &status_sender, - PolicyStatusUpdate::failed(result.version, e.to_string()), - ); - } - } - last_failed_runtime_revision = Some(failed_revision); - // Nothing was installed, so the registry status still - // describes the live registry. The retry is driven by the - // persisting hash/service-set mismatch (or an existing - // NeedsReconciliation), not by degrading the status here. - } - } - } - - // Apply OCSF JSON toggle from the `ocsf_json_enabled` setting. - apply_ocsf_json_setting(&ctx.ocsf_enabled, &result.settings); - - // Apply the agent-proposals feature toggle. On a false→true transition - // we lazily install the skill so a sandbox that started with the flag - // off picks up the surface without a recreate. We never uninstall on - // a true→false transition: stale skill content on disk is harmless - // because route_request and agent_next_steps both gate on the live - // shared flag, so the agent that reads the skill will see 404s and an - // empty `next_steps` array regardless. - apply_agent_proposals_enabled( - &ctx.agent_proposals, - agent_proposals_enabled_from_settings(&result.settings), - "settings poll", - Some(result.config_revision), - ctx.sidecar_control_publisher.as_ref(), - skills::install_static_skills, - ); - - current_config_revision = result.config_revision; - if !reloads_gateway_policy { - current_policy_hash = result.policy_hash; - } - current_settings = result.settings; - } -} - -fn apply_ocsf_json_setting( - enabled: &AtomicBool, - settings: &std::collections::HashMap, -) { - use std::sync::atomic::Ordering; - - let new_ocsf = extract_bool_setting(settings, "ocsf_json_enabled").unwrap_or(false); - let prev_ocsf = enabled.swap(new_ocsf, Ordering::Relaxed); - if new_ocsf != prev_ocsf { - info!(ocsf_json_enabled = new_ocsf, "OCSF JSONL logging toggled"); - } -} - -/// Extract a bool value from an effective setting, if present. -fn extract_bool_setting( - settings: &std::collections::HashMap, - key: &str, -) -> Option { - use openshell_core::proto::setting_value; - settings - .get(key) - .and_then(|es| es.value.as_ref()) - .and_then(|sv| sv.value.as_ref()) - .and_then(|v| match v { - setting_value::Value::BoolValue(b) => Some(*b), - _ => None, - }) -} - -fn agent_proposals_enabled_from_settings( - settings: &std::collections::HashMap, -) -> bool { - extract_bool_setting( - settings, - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY, - ) - .unwrap_or(false) -} - -fn apply_agent_proposals_enabled( - agent_proposals: &AgentProposals, - enabled: bool, - source: &'static str, - config_revision: Option, - sidecar_control_publisher: Option<&sidecar_control::Publisher>, - install_static_skills: impl FnOnce() -> Result, -) { - let previously_enabled = agent_proposals.swap_enabled(enabled); - if enabled == previously_enabled { - return; - } - - info!( - agent_policy_proposals_enabled = enabled, - source, config_revision, "agent-driven policy proposals toggled" - ); - - if let (Some(publisher), Some(config_revision)) = (sidecar_control_publisher, config_revision) { - publisher.publish_agent_proposals(enabled, config_revision); - } - - if enabled && !previously_enabled { - match install_static_skills() { - Ok(installed) => info!( - path = %installed.policy_advisor.display(), - "Installed sandbox agent skill on toggle-on" - ), - Err(error) => warn!( - error = %error, - "Failed to install sandbox agent skill on toggle-on" - ), - } - } -} - -/// Log individual setting changes between two snapshots. -fn log_setting_changes( - old: &std::collections::HashMap, - new: &std::collections::HashMap, -) { - for (key, new_es) in new { - let new_val = format_setting_value(new_es); - match old.get(key) { - Some(old_es) => { - let old_val = format_setting_value(old_es); - if old_val != new_val { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "updated") - .unmapped("key", serde_json::json!(key)) - .unmapped("old", serde_json::json!(old_val.clone())) - .unmapped("new", serde_json::json!(new_val.clone())) - .message(format!( - "Setting changed [key:{key} old:{old_val} new:{new_val}]" - )) - .build() - ); - } - } - None => { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Enabled, "enabled") - .unmapped("key", serde_json::json!(key)) - .unmapped("value", serde_json::json!(new_val.clone())) - .message(format!("Setting added [key:{key} value:{new_val}]")) - .build() - ); - } - } - } - for key in old.keys() { - if !new.contains_key(key) { - ocsf_emit!( - ConfigStateChangeBuilder::new(ocsf_ctx()) - .severity(SeverityId::Informational) - .status(StatusId::Success) - .state(StateId::Disabled, "disabled") - .unmapped("key", serde_json::json!(key)) - .message(format!("Setting removed [key:{key}]")) - .build() - ); - } - } -} - -/// Format an `EffectiveSetting` value for log display. -fn format_setting_value(es: &openshell_core::proto::EffectiveSetting) -> String { - use openshell_core::proto::setting_value; - match es.value.as_ref().and_then(|sv| sv.value.as_ref()) { - None => "".to_string(), - Some(setting_value::Value::StringValue(v)) => v.clone(), - Some(setting_value::Value::BoolValue(v)) => v.to_string(), - Some(setting_value::Value::IntValue(v)) => v.to_string(), - Some(setting_value::Value::BytesValue(_)) => "".to_string(), - } -} - -#[cfg(test)] -#[allow( - clippy::needless_raw_string_hashes, - clippy::iter_on_single_items, - clippy::similar_names, - clippy::manual_string_new, - clippy::doc_markdown, - reason = "Test code: test fixtures often use idiomatic forms not flagged in production." -)] -mod tests { - use super::*; - use openshell_core::policy::{ - FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, ProxyPolicy, - }; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; - - fn proxy_policy(http_addr: Option) -> SandboxPolicy { - SandboxPolicy { - version: 1, - filesystem: FilesystemPolicy::default(), - network: NetworkPolicy { - mode: NetworkMode::Proxy, - proxy: Some(ProxyPolicy { http_addr }), - }, - landlock: LandlockPolicy::default(), - process: ProcessPolicy::default(), - } - } - - fn effective_bool(value: bool) -> openshell_core::proto::EffectiveSetting { - openshell_core::proto::EffectiveSetting { - value: Some(openshell_core::proto::SettingValue { - value: Some(openshell_core::proto::setting_value::Value::BoolValue( - value, - )), - }), - scope: openshell_core::proto::SettingScope::Global.into(), - } - } - - #[test] - fn sidecar_process_policy_sets_loopback_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, true).unwrap(); - - let http_addr = process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .expect("sidecar process policy should set proxy address"); - assert_eq!(http_addr.to_string(), SIDECAR_PROCESS_PROXY_ADDR); - assert!( - policy - .network - .proxy - .as_ref() - .expect("original policy should keep proxy config") - .http_addr - .is_none(), - "process policy normalization must not mutate the network policy" - ); - } - - #[test] - fn non_sidecar_process_policy_preserves_proxy_addr() { - let policy = proxy_policy(None); - - let process_policy = process_policy_for_topology(&policy, false).unwrap(); - - assert!( - process_policy - .network - .proxy - .and_then(|proxy| proxy.http_addr) - .is_none() - ); - } - - #[tokio::test] - async fn sidecar_control_provider_env_update_installs_newer_revision() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = ProviderCredentialState::from_child_env_snapshot( - 1, - std::collections::HashMap::from([("TOKEN".to_string(), "old".to_string())]), - ); - let handle = spawn_sidecar_control_update_watcher( - rx, - provider_credentials.clone(), - AgentProposals::default(), - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 2, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "new".to_string(), - )]), - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if provider_credentials.snapshot().revision == 2 { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - let snapshot = provider_credentials.snapshot(); - assert_eq!(snapshot.revision, 2); - assert_eq!( - snapshot.child_env.get("TOKEN").map(String::as_str), - Some("new") - ); - - tx.send(sidecar_control::ControlUpdate::ProviderEnv { - revision: 1, - provider_child_env: std::collections::HashMap::from([( - "TOKEN".to_string(), - "stale".to_string(), - )]), - }) - .unwrap(); - tokio::time::sleep(Duration::from_millis(20)).await; - assert_eq!( - provider_credentials - .snapshot() - .child_env - .get("TOKEN") - .map(String::as_str), - Some("new") - ); - handle.abort(); - } - - #[tokio::test] - async fn sidecar_control_agent_proposals_update_flips_shared_state() { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let provider_credentials = - ProviderCredentialState::from_child_env_snapshot(0, std::collections::HashMap::new()); - let agent_proposals = AgentProposals::new(true); - let handle = - spawn_sidecar_control_update_watcher(rx, provider_credentials, agent_proposals.clone()); - - tx.send(sidecar_control::ControlUpdate::AgentProposals { - enabled: false, - config_revision: 5, - }) - .unwrap(); - - timeout(Duration::from_secs(1), async { - loop { - if !agent_proposals.enabled() { - break; - } - tokio::time::sleep(Duration::from_millis(10)).await; - } - }) - .await - .unwrap(); - handle.abort(); - } - - #[test] - fn apply_agent_proposals_enabled_installs_only_on_false_to_true() { - let agent_proposals = AgentProposals::default(); - let installs = AtomicUsize::new(0); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(1), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, true, "test", Some(2), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert_eq!(installs.load(Ordering::Relaxed), 1); - - apply_agent_proposals_enabled(&agent_proposals, false, "test", Some(3), None, || { - installs.fetch_add(1, Ordering::Relaxed); - Ok(skills::InstalledSkills { - policy_advisor: std::path::PathBuf::from("/tmp/policy_advisor.md"), - policy_advisor_skill: std::path::PathBuf::from("/tmp/SKILL.md"), - agents: None, - }) - }); - assert!(!agent_proposals.enabled()); - assert_eq!(installs.load(Ordering::Relaxed), 1); - } - - #[test] - fn apply_ocsf_json_setting_enables_from_initial_settings_snapshot() { - let enabled = AtomicBool::new(false); - let mut settings = std::collections::HashMap::new(); - settings.insert("ocsf_json_enabled".to_string(), effective_bool(true)); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(enabled.load(Ordering::Relaxed)); - } - - #[test] - fn apply_ocsf_json_setting_disables_when_setting_is_unset() { - let enabled = AtomicBool::new(true); - let settings = std::collections::HashMap::new(); - - apply_ocsf_json_setting(&enabled, &settings); - - assert!(!enabled.load(Ordering::Relaxed)); - } - - #[test] - fn agent_proposals_setting_enables_from_initial_settings_snapshot() { - let mut settings = std::collections::HashMap::new(); - settings.insert( - openshell_core::settings::AGENT_POLICY_PROPOSALS_ENABLED_KEY.to_string(), - effective_bool(true), - ); - - assert!(agent_proposals_enabled_from_settings(&settings)); - } - - #[test] - fn agent_proposals_setting_defaults_false_when_unset() { - let settings = std::collections::HashMap::new(); - - assert!(!agent_proposals_enabled_from_settings(&settings)); - } - - // ---- Policy disk discovery tests ---- - - #[test] - fn discover_policy_from_nonexistent_path_returns_restrictive_default() { - let path = std::path::Path::new("/nonexistent/policy.yaml"); - let policy = discover_policy_from_path(path); - // Restrictive default has no network policies. - assert!(policy.network_policies.is_empty()); - // But does have filesystem and process policies. - assert!(policy.filesystem.is_some()); - assert!(policy.process.is_some()); - } - - #[test] - fn discover_policy_from_valid_yaml_file() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -filesystem_policy: - include_workdir: false - read_only: - - /usr - read_write: - - /tmp -network_policies: - test: - name: test - endpoints: - - { host: example.com, port: 443 } - binaries: - - { path: /usr/bin/curl } -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - assert_eq!(policy.network_policies.len(), 1); - assert!(policy.network_policies.contains_key("test")); - let fs = policy.filesystem.unwrap(); - assert!(!fs.include_workdir); - } - - #[test] - fn discover_policy_from_invalid_yaml_returns_restrictive_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write(&path, "this is not valid yaml: [[[").unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default. - assert!(policy.network_policies.is_empty()); - assert!(policy.filesystem.is_some()); - } - - #[test] - fn discover_policy_from_unsafe_yaml_falls_back_to_default() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join("policy.yaml"); - std::fs::write( - &path, - r#" -version: 1 -process: - run_as_user: root - run_as_group: root -filesystem_policy: - include_workdir: true - read_only: - - /usr - read_write: - - /tmp -"#, - ) - .unwrap(); - - let policy = discover_policy_from_path(&path); - // Falls back to restrictive default because of root user. - let proc = policy.process.unwrap(); - assert_eq!(proc.run_as_user, "sandbox"); - assert_eq!(proc.run_as_group, "sandbox"); - } - - #[test] - fn discover_policy_restrictive_default_blocks_network() { - // In cluster mode we keep proxy mode enabled so `inference.local` - // can always be routed through proxy/OPA controls. - let proto = openshell_policy::restrictive_default_policy(); - let local_policy = SandboxPolicy::try_from(proto).expect("conversion should succeed"); - assert!(matches!(local_policy.network.mode, NetworkMode::Proxy)); - } - - // ---- Initial policy acknowledgement tests ---- - - fn proto_policy_fixture() -> openshell_core::proto::SandboxPolicy { - openshell_policy::restrictive_default_policy() - } - - fn settings_poll_result( - policy: Option, - version: u32, - source: openshell_core::proto::PolicySource, - ) -> openshell_core::grpc_client::SettingsPollResult { - openshell_core::grpc_client::SettingsPollResult { - policy, - version, - policy_hash: format!("hash-v{version}"), - config_revision: u64::from(version) * 100, - policy_source: source, - settings: std::collections::HashMap::new(), - global_policy_version: 0, - provider_env_revision: 0, - supervisor_middleware_services: Vec::new(), - workspace: String::new(), - } - } - - #[tokio::test] - async fn failed_external_startup_registry_build_preserves_installed_builtins() { - let engine = OpaEngine::from_proto(&proto_policy_fixture()).expect("build OPA engine"); - install_builtin_middleware_registry(&engine) - .await - .expect("install built-in middleware registry"); - let builtins_generation = engine.current_generation(); - assert_eq!(builtins_generation, 1); - - let invalid_external = openshell_core::proto::SupervisorMiddlewareService { - name: "unavailable-guard".into(), - grpc_endpoint: "http://127.0.0.1:1".into(), - max_body_bytes: 1024, - ..Default::default() - }; - connect_middleware_registry(&[invalid_external]) - .await - .expect_err("unavailable external service must not replace built-ins"); - - assert_eq!(engine.current_generation(), builtins_generation); - } - - #[test] - fn failed_gateway_runtime_snapshot_is_retried_without_revision_change() { - let services = Vec::new(); - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - } - - #[test] - fn gateway_runtime_reconciliation_tracks_policy_and_service_changes() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &no_services, - &no_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v1", - &no_services, - &desired_services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!gateway_policy_runtime_needs_reconciliation( - false, - "local-policy", - "hash-v2", - &no_services, - &desired_services, - MiddlewareRegistryStatus::NeedsReconciliation, - )); - } - - #[test] - fn policy_only_change_does_not_rebuild_middleware_registry() { - let services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - // The runtime must reconcile, but the registry (and therefore - // middleware reachability) is not part of that reconciliation. - assert!(gateway_policy_runtime_needs_reconciliation( - true, - "hash-v1", - "hash-v2", - &services, - &services, - MiddlewareRegistryStatus::Synchronized, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &services, - &services, - )); - } - - #[test] - fn registry_rebuild_requires_service_set_change_or_degraded_registry() { - let no_services = Vec::new(); - let desired_services = vec![openshell_core::proto::SupervisorMiddlewareService { - name: "guard".into(), - ..Default::default() - }]; - - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &no_services, - &desired_services, - )); - assert!(middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::NeedsReconciliation, - &desired_services, - &desired_services, - )); - assert!(!middleware_registry_needs_rebuild( - MiddlewareRegistryStatus::Synchronized, - &desired_services, - &desired_services, - )); - } - - #[test] - fn initial_ack_candidate_matches_sandbox_revision() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - let ack = initial_policy_ack_candidate(Some(&loaded), &canonical) - .expect("sandbox-sourced matching revision should be acknowledged"); - - assert_eq!(ack.version, 2); - assert_eq!(ack.policy_hash, "hash-v2"); - assert_eq!(ack.config_revision, 200); - } - - #[test] - fn initial_ack_candidate_ignores_global_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Global, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_version_zero() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 0, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&canonical); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_ignores_local_file_mode() { - // Local-file mode retains no proto policy, so there is nothing to - // acknowledge to the gateway. - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(None, &canonical).is_none()); - } - - #[test] - fn initial_ack_candidate_rejects_mismatched_identity() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert!(initial_policy_ack_candidate(Some(&loaded), &canonical).is_none()); - } - - #[test] - fn initial_poll_reconciles_provider_composition_that_was_not_loaded() { - let loaded_snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - let loaded = LoadedPolicyRevision::from_snapshot(&loaded_snapshot); - let mut newer = proto_policy_fixture(); - newer.network_policies.insert( - "_provider_work_github".to_string(), - openshell_core::proto::NetworkPolicyRule::default(), - ); - let canonical = - settings_poll_result(Some(newer), 1, openshell_core::proto::PolicySource::Sandbox); - let canonical = openshell_core::grpc_client::SettingsPollResult { - policy_hash: "hash-provider-change".to_string(), - config_revision: loaded.config_revision + 1, - ..canonical - }; - - assert_eq!( - initial_poll_disposition( - &LoadedPolicyOrigin::Gateway { - revision: Some(loaded), - }, - &canonical, - ), - InitialPollDisposition::Reconcile - ); - } - - #[test] - fn initial_poll_tracks_local_override_without_reconciliation() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - - assert_eq!( - initial_poll_disposition(&LoadedPolicyOrigin::LocalOverride, &canonical), - InitialPollDisposition::TrackOnly - ); - assert!(!LoadedPolicyOrigin::LocalOverride.allows_gateway_policy_reload()); - } - - #[test] - fn initial_poll_reconciles_unbound_gateway_policy() { - let canonical = settings_poll_result( - Some(proto_policy_fixture()), - 2, - openshell_core::proto::PolicySource::Sandbox, - ); - let origin = LoadedPolicyOrigin::Gateway { revision: None }; - - assert_eq!( - initial_poll_disposition(&origin, &canonical), - InitialPollDisposition::Reconcile - ); - assert!(origin.allows_gateway_policy_reload()); - } - - #[test] - fn policy_status_outbox_preserves_all_revision_order() { - let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); - for version in 1..=128 { - enqueue_policy_status(&sender, PolicyStatusUpdate::loaded(version)); - } - - for version in 1..=128 { - assert_eq!( - receiver.try_recv().unwrap(), - PolicyStatusUpdate::loaded(version) - ); - } - } - - #[test] - fn settings_snapshot_carries_workspace_for_policy_sync() { - let mut snapshot = settings_poll_result( - Some(proto_policy_fixture()), - 1, - openshell_core::proto::PolicySource::Sandbox, - ); - snapshot.workspace = "beta".to_string(); - - let revision = LoadedPolicyRevision::from_snapshot(&snapshot); - assert_eq!(revision.version, 1); - assert_eq!( - snapshot.workspace, "beta", - "workspace must survive the snapshot so sync_policy_and_fetch_snapshot receives it" - ); - } -} diff --git a/crates/openshell-sandbox/src/lib_win.rs b/crates/openshell-sandbox/src/lib_win.rs new file mode 100644 index 000000000..394226e1a --- /dev/null +++ b/crates/openshell-sandbox/src/lib_win.rs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +pub fn windows_unsupported() -> &'static str { + "openshell-sandbox supervisor is not available on Windows" +} diff --git a/crates/openshell-sandbox/src/main.rs b/crates/openshell-sandbox/src/main.rs index 948e0e710..98af7f9ea 100644 --- a/crates/openshell-sandbox/src/main.rs +++ b/crates/openshell-sandbox/src/main.rs @@ -1,11 +1,822 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -#[cfg(target_os = "windows")] -fn main() { - eprintln!("openshell-sandbox supervisor is not available on Windows"); - std::process::exit(1); +//! `OpenShell` Sandbox - process sandbox and monitor. + +use std::path::Path; +use std::sync::Arc; +use std::sync::atomic::AtomicBool; + +use clap::Parser; +use miette::{IntoDiagnostic, Result}; +use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; +use tracing::{info, warn}; +use tracing_subscriber::EnvFilter; +use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; + +use openshell_sandbox::run_sandbox; + +/// Subcommand name used to self-copy the supervisor binary into a shared volume. +/// +/// Init containers invoke the binary directly instead of relying on `sh`/`cp` +/// to copy the binary out. Invoking the binary itself with this argument +/// performs the copy in pure Rust. +const COPY_SELF_SUBCOMMAND: &str = "copy-self"; + +/// Subcommand for one-shot debug RPCs from inside a sandbox container. +/// +/// Reads the same token sources as the supervisor (env, file, K8s SA +/// bootstrap) and issues a single gRPC call against the gateway. Useful +/// for end-to-end verification: e.g. `docker exec` into a sandbox, then +/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` +/// to confirm the cross-sandbox IDOR guard fires. +const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; +const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; + +/// Default `--mode` value: run both supervisor leaves in a single binary. +const DEFAULT_MODE: &str = "network,process"; +const SIDECAR_STATE_DIR: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; +const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; +#[cfg(target_os = "linux")] +const CLIENT_TLS_DIR: &str = openshell_core::container_paths::CLIENT_TLS_DIR; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; +#[cfg(target_os = "linux")] +const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; +#[cfg(target_os = "linux")] +const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_DIR_MODE: u32 = 0o755; +#[cfg(target_os = "linux")] +const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; +#[cfg(target_os = "linux")] +const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; + +/// Which supervisor leaves are enabled in this process. +/// +/// Parsed from a comma-separated `--mode` value, e.g. `network`, +/// `process`, or `network,process`. `network-init` is a one-shot setup mode +/// used by the Kubernetes sidecar topology and cannot be combined with other +/// mode components. At least one must be set. +#[derive(Clone, Copy, Debug)] +struct Mode { + network: bool, + process: bool, + network_init: bool, } -#[cfg(not(target_os = "windows"))] -include!("main_unix.rs"); +impl std::str::FromStr for Mode { + type Err = String; + + fn from_str(s: &str) -> Result { + let mut mode = Self { + network: false, + process: false, + network_init: false, + }; + for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { + match part { + "network" => mode.network = true, + "process" => mode.process = true, + "network-init" => mode.network_init = true, + other => { + return Err(format!( + "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" + )); + } + } + } + if mode.network_init && (mode.network || mode.process) { + return Err("--mode=network-init cannot be combined with other components".into()); + } + if !mode.network && !mode.process && !mode.network_init { + return Err( + "--mode must enable at least one of: network, process, network-init".into(), + ); + } + Ok(mode) + } +} + +/// `OpenShell` Sandbox - process isolation and monitoring. +// CLI flags are naturally boolean switches; grouping them into structs would +// only obscure the clap definition. +#[allow(clippy::struct_excessive_bools)] +#[derive(Parser, Debug)] +#[command(name = "openshell-sandbox")] +#[command(version = openshell_core::VERSION)] +#[command(about = "Process sandbox and monitor", long_about = None)] +struct Args { + /// Command to execute in the sandbox. + /// Can also be provided via `OPENSHELL_SANDBOX_COMMAND` environment variable. + /// Defaults to `/bin/bash` if neither is provided. + #[arg(trailing_var_arg = true)] + command: Vec, + + /// Working directory for the sandboxed process. + #[arg(long, short)] + workdir: Option, + + /// Timeout in seconds (0 = no timeout). + #[arg(long, short, default_value = "0")] + timeout: u64, + + /// Run in interactive mode (inherit process group for terminal control). + #[arg(long, short = 'i')] + interactive: bool, + + /// Sandbox ID for fetching policy via gRPC from `OpenShell` server. + /// Requires --openshell-endpoint to be set. + #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] + sandbox_id: Option, + + /// Sandbox (used for policy sync when the sandbox discovers policy + /// from disk or falls back to the restrictive default). + #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] + sandbox: Option, + + /// `OpenShell` server gRPC endpoint for fetching policy. + /// Required when using --sandbox-id. + #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] + openshell_endpoint: Option, + + /// Path to Rego policy file for OPA-based network access control. + /// Requires --policy-data to also be set. + #[arg(long, env = "OPENSHELL_POLICY_RULES")] + policy_rules: Option, + + /// Path to YAML data file containing network policies and sandbox config. + /// Requires --policy-rules to also be set. + #[arg(long, env = "OPENSHELL_POLICY_DATA")] + policy_data: Option, + + /// Log level (trace, debug, info, warn, error). + #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] + log_level: String, + + /// Unix socket the embedded SSH daemon binds. On Linux, a value beginning + /// with `@` selects an abstract socket in the network namespace. + /// The supervisor bridges `RelayStream` traffic from the gateway onto + /// this socket; nothing else should connect to it. + #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] + ssh_socket_path: Option, + + /// Path to YAML inference routes for standalone routing. + /// When set, inference routes are loaded from this file instead of + /// fetching a bundle from the gateway. + #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] + inference_routes: Option, + + /// Enable health check endpoint. + #[arg(long)] + health_check: bool, + + /// Port for health check endpoint. + #[arg(long, default_value = "8080")] + health_port: u16, + + /// Which supervisor components to run. Comma-separated list of + /// "network" and/or "process". Defaults to both (single-binary + /// topology). Use --mode=network for a network-only sidecar, or + /// --mode=process for a process-only supervisor when network + /// enforcement runs in another pod. Use --mode=network-init only in + /// the Kubernetes init container that prepares sidecar nftables. + #[arg(long, default_value = DEFAULT_MODE)] + mode: Mode, + + /// UID that the long-running Kubernetes network sidecar will run as. + /// `--mode=network-init` installs nftables rules that exempt this UID. + #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] + proxy_uid: u32, + + /// GID assigned to shared sidecar state directories. Defaults to + /// `--proxy-uid` when omitted. + #[arg(long, env = "OPENSHELL_PROXY_GID")] + proxy_gid: Option, + + /// Shared state directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] + sidecar_state_dir: String, + + /// Shared TLS work directory between the network init container and sidecar. + #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] + sidecar_tls_dir: String, + + // Corporate upstream proxy. Operator-owned egress boundary: accepted + // only as command-line arguments (no `env =`), because the driver + // controls the supervisor's argv while a sandbox image could bake + // matching `ENV` values. + /// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress. + #[arg(long)] + upstream_proxy: Option, + + /// Comma-separated `NO_PROXY` list for the corporate proxy. + #[arg(long)] + upstream_no_proxy: Option, + + /// Path to the root-only file holding corporate proxy credentials (`user:pass`). + #[arg(long)] + upstream_proxy_auth_file: Option, + + /// Acknowledge that proxy credentials travel as cleartext Basic auth over + /// the plain-TCP connection to the `http://` proxy. + #[arg(long)] + upstream_proxy_auth_allow_insecure: bool, + + /// Send the destination hostname in CONNECT instead of a validated IP + /// (for proxies whose ACLs filter on hostnames). + #[arg(long)] + upstream_proxy_connect_by_hostname: bool, +} + +/// Internal one-shot command used by the privileged supervisor to validate an +/// image-provided workdir as the final sandbox identity. +#[derive(Parser, Debug)] +#[command(name = "validate-workspace", hide = true)] +struct ValidateWorkspaceArgs { + #[arg(long)] + workdir: String, + #[arg(long)] + expected_uid: u32, + #[arg(long)] + expected_gid: u32, +} + +#[cfg(target_os = "linux")] +fn validate_workspace(args: &[String]) -> Result<()> { + let args = ValidateWorkspaceArgs::try_parse_from( + std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), + ) + .into_diagnostic()?; + let actual = ( + nix::unistd::geteuid().as_raw(), + nix::unistd::getegid().as_raw(), + ); + if actual != (args.expected_uid, args.expected_gid) { + return Err(miette::miette!( + "workspace validator privilege drop failed: expected {}:{}, got {}:{}", + args.expected_uid, + args.expected_gid, + actual.0, + actual.1 + )); + } + openshell_supervisor_process::process::validate_oci_workspace_as_effective_identity(Path::new( + &args.workdir, + )) +} + +#[cfg(not(target_os = "linux"))] +fn validate_workspace(_args: &[String]) -> Result<()> { + Err(miette::miette!( + "workspace validation is only supported on Unix" + )) +} + +/// Copy the running executable to `dest`, creating parent directories as +/// needed and ensuring the result is executable (mode `0755`). +/// +/// If `dest` already exists as a directory, the binary is placed inside it +/// using the source executable's file name. This mirrors `cp` semantics so +/// callers can pass either a full target path or a directory. +fn copy_self(dest: &str) -> Result<()> { + let exe = std::env::current_exe().into_diagnostic()?; + + let dest_path = Path::new(dest); + let final_path = if dest_path.is_dir() { + let file_name = exe + .file_name() + .ok_or_else(|| miette::miette!("current_exe has no file name: {}", exe.display()))?; + dest_path.join(file_name) + } else { + dest_path.to_path_buf() + }; + + if let Some(parent) = final_path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + + std::fs::copy(&exe, &final_path).into_diagnostic()?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&final_path) + .into_diagnostic()? + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&final_path, perms).into_diagnostic()?; + } + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {uid}:{gid}", + path.display() + ) + })?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + let uid = Uid::current(); + let gid = Gid::current(); + std::fs::create_dir_all(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; + chown(path, Some(uid), Some(gid)) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown sidecar directory {} to {}:{}", + path.display(), + uid.as_raw(), + gid.as_raw() + ) + })?; + let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); + perms.set_mode(mode); + std::fs::set_permissions(path, perms) + .into_diagnostic() + .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; + Ok(()) +} + +#[cfg(target_os = "linux")] +fn copy_sidecar_client_tls_if_present( + source_dir: &Path, + sidecar_tls_dir: &Path, + uid: u32, + gid: u32, +) -> Result<()> { + use miette::Context as _; + use nix::unistd::{Gid, Uid, chown}; + use std::os::unix::fs::PermissionsExt; + + if !source_dir.exists() { + return Ok(()); + } + + let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); + prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; + for file_name in CLIENT_TLS_FILES { + let source = source_dir.join(file_name); + if !source.exists() { + return Err(miette::miette!( + "client TLS source file is missing: {}", + source.display() + )); + } + let dest = dest_dir.join(file_name); + if dest.exists() { + std::fs::remove_file(&dest) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to remove stale client TLS file {}", dest.display()) + })?; + } + std::fs::copy(&source, &dest) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to copy client TLS file {} to {}", + source.display(), + dest.display() + ) + })?; + let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); + perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); + std::fs::set_permissions(&dest, perms) + .into_diagnostic() + .wrap_err_with(|| { + format!("failed to chmod copied client TLS file {}", dest.display()) + })?; + chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) + .into_diagnostic() + .wrap_err_with(|| { + format!( + "failed to chown copied client TLS file {} to {uid}:{gid}", + dest.display() + ) + })?; + } + + prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; + + Ok(()) +} + +#[cfg(target_os = "linux")] +fn run_network_init( + proxy_user_id: u32, + proxy_primary_group_id: u32, + sidecar_state_dir: &str, + sidecar_tls_dir: &str, +) -> Result<()> { + validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; + + let sidecar_state_dir = Path::new(sidecar_state_dir); + let sidecar_tls_dir = Path::new(sidecar_tls_dir); + prepare_sidecar_directory( + sidecar_state_dir, + proxy_user_id, + proxy_primary_group_id, + SIDECAR_STATE_DIR_MODE, + )?; + // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the + // TLS work directory owned by the init user until the client cert copy is + // complete, then hand it to the long-running proxy UID. + prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; + copy_sidecar_client_tls_if_present( + Path::new(CLIENT_TLS_DIR), + sidecar_tls_dir, + proxy_user_id, + proxy_primary_group_id, + )?; + prepare_sidecar_directory( + sidecar_tls_dir, + proxy_user_id, + proxy_primary_group_id, + SIDECAR_TLS_DIR_MODE, + )?; + openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) +} + +#[cfg(target_os = "linux")] +fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { + if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-uid must be 0 or at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { + return Err(miette::miette!( + "--proxy-gid must be at least {}", + openshell_policy::MIN_SANDBOX_UID + )); + } + Ok(()) +} + +#[cfg(not(target_os = "linux"))] +fn run_network_init( + _proxy_uid: u32, + _proxy_gid: u32, + _sidecar_state_dir: &str, + _sidecar_tls_dir: &str, +) -> Result<()> { + Err(miette::miette!( + "--mode=network-init is only supported on Linux" + )) +} + +fn main() -> Result<()> { + // Handle `copy-self ` before clap so it works without any of the + // sandbox flags. Kubernetes init containers invoke this path to seed an + // emptyDir volume that the agent container then executes from. + let raw_args: Vec = std::env::args().collect(); + if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) { + let dest = raw_args.get(2).ok_or_else(|| { + miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} ") + })?; + return copy_self(dest); + } + + // Handle `debug-rpc [args]` before clap. Uses a small + // dedicated runtime so we don't pay the supervisor's full startup cost. + if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .into_diagnostic()?; + return runtime.block_on(async move { + let _ = rustls::crypto::ring::default_provider().install_default(); + let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; + std::process::exit(exit); + }); + } + if raw_args.get(1).map(String::as_str) == Some(VALIDATE_WORKSPACE_SUBCOMMAND) { + return validate_workspace(&raw_args[2..]); + } + + let args = Args::parse(); + + if args.mode.network_init { + let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); + return run_network_init( + args.proxy_uid, + proxy_gid, + &args.sidecar_state_dir, + &args.sidecar_tls_dir, + ); + } + + // Try to open a rolling log file; fall back to stderr-only logging if it fails + // (e.g., /var/log is not writable in custom workload images). + // Rotates daily, keeps the 3 most recent files to bound disk usage. + let file_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + (writer, guard) + }); + + let console_filter = + EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); + + let runtime = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build() + .into_diagnostic()?; + + let exit_code = runtime.block_on(async move { + // Install rustls crypto provider before any TLS connections (including log push). + let _ = rustls::crypto::ring::default_provider().install_default(); + + // Set up optional log push layer (gRPC mode only). + let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = + (&args.sandbox_id, &args.openshell_endpoint) + { + let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( + endpoint.clone(), + sandbox_id.clone(), + ); + let layer = + openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); + Some((layer, handle)) + } else { + None + }; + let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); + let _log_push_handle = log_push_state.map(|(_, handle)| handle); + + // Shared flag: the sandbox poll loop toggles this when the + // `ocsf_json_enabled` setting changes. The JSONL layer checks it + // on each event and short-circuits when false. + let ocsf_enabled = Arc::new(AtomicBool::new(false)); + + // Keep guards alive for the entire process. When a guard is dropped the + // non-blocking writer flushes remaining logs. + let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { + let file_filter = EnvFilter::new("info"); + + // OCSF JSONL file: rolling appender matching the main log file + // (daily rotation, 3 files max). Created eagerly but gated by the + // enabled flag — no JSONL is written until ocsf_json_enabled is set. + let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() + .rotation(tracing_appender::rolling::Rotation::DAILY) + .filename_prefix("openshell-ocsf") + .filename_suffix("log") + .max_log_files(3) + .build("/var/log") + .ok() + .map(|roller| { + let (writer, guard) = tracing_appender::non_blocking(roller); + let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); + (layer, guard) + }); + let (jsonl_layer, jsonl_guard) = match jsonl_logging { + Some((layer, guard)) => (Some(layer), Some(guard)), + None => (None, None), + }; + + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with( + OcsfShorthandLayer::new(file_writer) + .with_non_ocsf(true) + .with_filter(file_filter), + ) + .with(jsonl_layer.with_filter(LevelFilter::INFO)) + .with(push_layer.clone()) + .init(); + (Some(file_guard), jsonl_guard) + } else { + tracing_subscriber::registry() + .with( + OcsfShorthandLayer::new(std::io::stderr()) + .with_non_ocsf(true) + .with_filter(console_filter), + ) + .with(push_layer) + .init(); + // Log the warning after the subscriber is initialized + warn!("Could not open /var/log for log rotation; using stderr-only logging"); + (None, None) + }; + + // Get command - either from CLI args, environment variable, or default to /bin/bash + let command = if !args.command.is_empty() { + args.command + } else if let Ok(c) = std::env::var(openshell_core::sandbox_env::SANDBOX_COMMAND) { + // Simple shell-like splitting on whitespace + c.split_whitespace().map(String::from).collect() + } else { + vec!["/bin/bash".to_string()] + }; + + info!(command = ?command, "Starting sandbox"); + // Note: "Starting sandbox" stays as plain info!() since the OCSF context + // is not yet initialized at this point (run_sandbox hasn't been called). + // The shorthand layer will render it in fallback format. + + let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { + https_proxy: args.upstream_proxy, + no_proxy: args.upstream_no_proxy, + proxy_auth_file: args.upstream_proxy_auth_file, + proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, + proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, + }; + + run_sandbox( + command, + args.workdir, + args.timeout, + args.interactive, + args.sandbox_id, + args.sandbox, + args.openshell_endpoint, + args.policy_rules, + args.policy_data, + args.ssh_socket_path, + args.health_check, + args.health_port, + args.inference_routes, + ocsf_enabled, + args.mode.network, + args.mode.process, + upstream_proxy_args, + ) + .await + })?; + + std::process::exit(exit_code); +} + +#[cfg(test)] +mod tests { + use super::*; + use std::os::unix::fs::PermissionsExt; + + #[cfg(target_os = "linux")] + #[test] + fn workspace_validation_subcommand_uses_final_policy_identity() { + let uid = nix::unistd::geteuid().as_raw(); + let gid = nix::unistd::getegid().as_raw(); + if uid < 1000 || gid < 1000 { + return; + } + let dir = tempfile::tempdir_in("/tmp").unwrap(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); + let root = dir.path().canonicalize().unwrap().join("workspace"); + std::fs::create_dir(&root).unwrap(); + std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); + let args = vec![ + "--workdir".to_string(), + root.display().to_string(), + "--expected-uid".to_string(), + uid.to_string(), + "--expected-gid".to_string(), + gid.to_string(), + ]; + + validate_workspace(&args).expect("current identity should retain workspace authority"); + } + + /// Drives `copy_self`'s file-copy logic against an arbitrary source path + /// so tests don't depend on `current_exe()`. + fn copy_executable(src: &Path, dest: &Path) -> Result<()> { + let final_path = if dest.is_dir() { + dest.join(src.file_name().unwrap()) + } else { + dest.to_path_buf() + }; + if let Some(parent) = final_path.parent() + && !parent.as_os_str().is_empty() + { + std::fs::create_dir_all(parent).into_diagnostic()?; + } + std::fs::copy(src, &final_path).into_diagnostic()?; + let mut perms = std::fs::metadata(&final_path) + .into_diagnostic()? + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&final_path, perms).into_diagnostic()?; + Ok(()) + } + + #[test] + fn copy_self_writes_executable_at_target_path() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("source-bin"); + std::fs::write(&src, b"#!/bin/false\n").unwrap(); + + let dest = tmp.path().join("subdir/openshell-sandbox"); + copy_executable(&src, &dest).unwrap(); + + assert!(dest.exists(), "destination file should exist"); + let mode = std::fs::metadata(&dest).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o755, "destination must be 0755"); + let copied = std::fs::read(&dest).unwrap(); + assert_eq!(copied, b"#!/bin/false\n"); + } + + #[test] + fn copy_self_into_existing_directory_uses_source_filename() { + let tmp = tempfile::tempdir().unwrap(); + let src = tmp.path().join("openshell-sandbox"); + std::fs::write(&src, b"binary").unwrap(); + + let dest_dir = tmp.path().join("bin"); + std::fs::create_dir_all(&dest_dir).unwrap(); + + copy_executable(&src, &dest_dir).unwrap(); + + let final_path = dest_dir.join("openshell-sandbox"); + assert!(final_path.exists(), "binary should land inside dest dir"); + } + + #[test] + fn mode_parses_network_init_standalone() { + let mode = "network-init".parse::().unwrap(); + assert!(mode.network_init); + assert!(!mode.network); + assert!(!mode.process); + } + + #[test] + fn mode_rejects_combined_network_init() { + let err = "network-init,network".parse::().unwrap_err(); + assert!(err.contains("cannot be combined")); + } + + #[test] + fn mode_rejects_empty_value() { + let err = "".parse::().unwrap_err(); + assert!(err.contains("at least one")); + } + + #[cfg(target_os = "linux")] + #[test] + fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { + assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); + assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); + assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); + assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); + } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { + validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap(); + } + + #[cfg(target_os = "linux")] + #[test] + fn network_init_still_rejects_low_non_root_proxy_ids() { + let uid_err = + validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); + assert!(uid_err.to_string().contains("--proxy-uid")); + + let gid_err = validate_network_init_ids(0, 999).unwrap_err(); + assert!(gid_err.to_string().contains("--proxy-gid")); + } +} diff --git a/crates/openshell-sandbox/src/main_entry.rs b/crates/openshell-sandbox/src/main_entry.rs new file mode 100644 index 000000000..432402121 --- /dev/null +++ b/crates/openshell-sandbox/src/main_entry.rs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +#[cfg(not(target_os = "windows"))] +include!("main.rs"); + +#[cfg(target_os = "windows")] +include!("main_win.rs"); diff --git a/crates/openshell-sandbox/src/main_unix.rs b/crates/openshell-sandbox/src/main_unix.rs deleted file mode 100644 index 98af7f9ea..000000000 --- a/crates/openshell-sandbox/src/main_unix.rs +++ /dev/null @@ -1,822 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -//! `OpenShell` Sandbox - process sandbox and monitor. - -use std::path::Path; -use std::sync::Arc; -use std::sync::atomic::AtomicBool; - -use clap::Parser; -use miette::{IntoDiagnostic, Result}; -use openshell_ocsf::{OcsfJsonlLayer, OcsfShorthandLayer}; -use tracing::{info, warn}; -use tracing_subscriber::EnvFilter; -use tracing_subscriber::filter::LevelFilter; -use tracing_subscriber::{Layer, layer::SubscriberExt, util::SubscriberInitExt}; - -use openshell_sandbox::run_sandbox; - -/// Subcommand name used to self-copy the supervisor binary into a shared volume. -/// -/// Init containers invoke the binary directly instead of relying on `sh`/`cp` -/// to copy the binary out. Invoking the binary itself with this argument -/// performs the copy in pure Rust. -const COPY_SELF_SUBCOMMAND: &str = "copy-self"; - -/// Subcommand for one-shot debug RPCs from inside a sandbox container. -/// -/// Reads the same token sources as the supervisor (env, file, K8s SA -/// bootstrap) and issues a single gRPC call against the gateway. Useful -/// for end-to-end verification: e.g. `docker exec` into a sandbox, then -/// run `openshell-sandbox debug-rpc get-sandbox-config --sandbox-id ` -/// to confirm the cross-sandbox IDOR guard fires. -const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc"; -const VALIDATE_WORKSPACE_SUBCOMMAND: &str = "validate-workspace"; - -/// Default `--mode` value: run both supervisor leaves in a single binary. -const DEFAULT_MODE: &str = "network,process"; -const SIDECAR_STATE_DIR: &str = openshell_core::container_paths::SIDECAR_RUN_ROOT; -const SIDECAR_TLS_DIR: &str = openshell_core::container_paths::SIDECAR_TLS_DIR; -#[cfg(target_os = "linux")] -const CLIENT_TLS_DIR: &str = openshell_core::container_paths::CLIENT_TLS_DIR; -#[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_SUBDIR: &str = "client"; -#[cfg(target_os = "linux")] -const CLIENT_TLS_FILES: [&str; 3] = ["ca.crt", "tls.crt", "tls.key"]; -#[cfg(target_os = "linux")] -const SIDECAR_STATE_DIR_MODE: u32 = 0o2775; -#[cfg(target_os = "linux")] -const SIDECAR_TLS_DIR_MODE: u32 = 0o755; -#[cfg(target_os = "linux")] -const SIDECAR_TLS_STAGING_DIR_MODE: u32 = 0o700; -#[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_DIR_MODE: u32 = 0o750; -#[cfg(target_os = "linux")] -const SIDECAR_CLIENT_TLS_FILE_MODE: u32 = 0o400; - -/// Which supervisor leaves are enabled in this process. -/// -/// Parsed from a comma-separated `--mode` value, e.g. `network`, -/// `process`, or `network,process`. `network-init` is a one-shot setup mode -/// used by the Kubernetes sidecar topology and cannot be combined with other -/// mode components. At least one must be set. -#[derive(Clone, Copy, Debug)] -struct Mode { - network: bool, - process: bool, - network_init: bool, -} - -impl std::str::FromStr for Mode { - type Err = String; - - fn from_str(s: &str) -> Result { - let mut mode = Self { - network: false, - process: false, - network_init: false, - }; - for part in s.split(',').map(str::trim).filter(|p| !p.is_empty()) { - match part { - "network" => mode.network = true, - "process" => mode.process = true, - "network-init" => mode.network_init = true, - other => { - return Err(format!( - "unknown mode component '{other}' (expected 'network', 'process', or 'network-init')" - )); - } - } - } - if mode.network_init && (mode.network || mode.process) { - return Err("--mode=network-init cannot be combined with other components".into()); - } - if !mode.network && !mode.process && !mode.network_init { - return Err( - "--mode must enable at least one of: network, process, network-init".into(), - ); - } - Ok(mode) - } -} - -/// `OpenShell` Sandbox - process isolation and monitoring. -// CLI flags are naturally boolean switches; grouping them into structs would -// only obscure the clap definition. -#[allow(clippy::struct_excessive_bools)] -#[derive(Parser, Debug)] -#[command(name = "openshell-sandbox")] -#[command(version = openshell_core::VERSION)] -#[command(about = "Process sandbox and monitor", long_about = None)] -struct Args { - /// Command to execute in the sandbox. - /// Can also be provided via `OPENSHELL_SANDBOX_COMMAND` environment variable. - /// Defaults to `/bin/bash` if neither is provided. - #[arg(trailing_var_arg = true)] - command: Vec, - - /// Working directory for the sandboxed process. - #[arg(long, short)] - workdir: Option, - - /// Timeout in seconds (0 = no timeout). - #[arg(long, short, default_value = "0")] - timeout: u64, - - /// Run in interactive mode (inherit process group for terminal control). - #[arg(long, short = 'i')] - interactive: bool, - - /// Sandbox ID for fetching policy via gRPC from `OpenShell` server. - /// Requires --openshell-endpoint to be set. - #[arg(long, env = openshell_core::sandbox_env::SANDBOX_ID)] - sandbox_id: Option, - - /// Sandbox (used for policy sync when the sandbox discovers policy - /// from disk or falls back to the restrictive default). - #[arg(long, env = openshell_core::sandbox_env::SANDBOX)] - sandbox: Option, - - /// `OpenShell` server gRPC endpoint for fetching policy. - /// Required when using --sandbox-id. - #[arg(long, env = openshell_core::sandbox_env::ENDPOINT)] - openshell_endpoint: Option, - - /// Path to Rego policy file for OPA-based network access control. - /// Requires --policy-data to also be set. - #[arg(long, env = "OPENSHELL_POLICY_RULES")] - policy_rules: Option, - - /// Path to YAML data file containing network policies and sandbox config. - /// Requires --policy-rules to also be set. - #[arg(long, env = "OPENSHELL_POLICY_DATA")] - policy_data: Option, - - /// Log level (trace, debug, info, warn, error). - #[arg(long, default_value = "warn", env = openshell_core::sandbox_env::LOG_LEVEL)] - log_level: String, - - /// Unix socket the embedded SSH daemon binds. On Linux, a value beginning - /// with `@` selects an abstract socket in the network namespace. - /// The supervisor bridges `RelayStream` traffic from the gateway onto - /// this socket; nothing else should connect to it. - #[arg(long, env = openshell_core::sandbox_env::SSH_SOCKET_PATH)] - ssh_socket_path: Option, - - /// Path to YAML inference routes for standalone routing. - /// When set, inference routes are loaded from this file instead of - /// fetching a bundle from the gateway. - #[arg(long, env = "OPENSHELL_INFERENCE_ROUTES")] - inference_routes: Option, - - /// Enable health check endpoint. - #[arg(long)] - health_check: bool, - - /// Port for health check endpoint. - #[arg(long, default_value = "8080")] - health_port: u16, - - /// Which supervisor components to run. Comma-separated list of - /// "network" and/or "process". Defaults to both (single-binary - /// topology). Use --mode=network for a network-only sidecar, or - /// --mode=process for a process-only supervisor when network - /// enforcement runs in another pod. Use --mode=network-init only in - /// the Kubernetes init container that prepares sidecar nftables. - #[arg(long, default_value = DEFAULT_MODE)] - mode: Mode, - - /// UID that the long-running Kubernetes network sidecar will run as. - /// `--mode=network-init` installs nftables rules that exempt this UID. - #[arg(long, env = "OPENSHELL_PROXY_UID", default_value_t = 1337)] - proxy_uid: u32, - - /// GID assigned to shared sidecar state directories. Defaults to - /// `--proxy-uid` when omitted. - #[arg(long, env = "OPENSHELL_PROXY_GID")] - proxy_gid: Option, - - /// Shared state directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_SIDECAR_STATE_DIR", default_value = SIDECAR_STATE_DIR)] - sidecar_state_dir: String, - - /// Shared TLS work directory between the network init container and sidecar. - #[arg(long, env = "OPENSHELL_PROXY_TLS_DIR", default_value = SIDECAR_TLS_DIR)] - sidecar_tls_dir: String, - - // Corporate upstream proxy. Operator-owned egress boundary: accepted - // only as command-line arguments (no `env =`), because the driver - // controls the supervisor's argv while a sandbox image could bake - // matching `ENV` values. - /// Corporate forward proxy URL (`http://host:port`) for upstream TLS egress. - #[arg(long)] - upstream_proxy: Option, - - /// Comma-separated `NO_PROXY` list for the corporate proxy. - #[arg(long)] - upstream_no_proxy: Option, - - /// Path to the root-only file holding corporate proxy credentials (`user:pass`). - #[arg(long)] - upstream_proxy_auth_file: Option, - - /// Acknowledge that proxy credentials travel as cleartext Basic auth over - /// the plain-TCP connection to the `http://` proxy. - #[arg(long)] - upstream_proxy_auth_allow_insecure: bool, - - /// Send the destination hostname in CONNECT instead of a validated IP - /// (for proxies whose ACLs filter on hostnames). - #[arg(long)] - upstream_proxy_connect_by_hostname: bool, -} - -/// Internal one-shot command used by the privileged supervisor to validate an -/// image-provided workdir as the final sandbox identity. -#[derive(Parser, Debug)] -#[command(name = "validate-workspace", hide = true)] -struct ValidateWorkspaceArgs { - #[arg(long)] - workdir: String, - #[arg(long)] - expected_uid: u32, - #[arg(long)] - expected_gid: u32, -} - -#[cfg(target_os = "linux")] -fn validate_workspace(args: &[String]) -> Result<()> { - let args = ValidateWorkspaceArgs::try_parse_from( - std::iter::once(VALIDATE_WORKSPACE_SUBCOMMAND.to_string()).chain(args.iter().cloned()), - ) - .into_diagnostic()?; - let actual = ( - nix::unistd::geteuid().as_raw(), - nix::unistd::getegid().as_raw(), - ); - if actual != (args.expected_uid, args.expected_gid) { - return Err(miette::miette!( - "workspace validator privilege drop failed: expected {}:{}, got {}:{}", - args.expected_uid, - args.expected_gid, - actual.0, - actual.1 - )); - } - openshell_supervisor_process::process::validate_oci_workspace_as_effective_identity(Path::new( - &args.workdir, - )) -} - -#[cfg(not(target_os = "linux"))] -fn validate_workspace(_args: &[String]) -> Result<()> { - Err(miette::miette!( - "workspace validation is only supported on Unix" - )) -} - -/// Copy the running executable to `dest`, creating parent directories as -/// needed and ensuring the result is executable (mode `0755`). -/// -/// If `dest` already exists as a directory, the binary is placed inside it -/// using the source executable's file name. This mirrors `cp` semantics so -/// callers can pass either a full target path or a directory. -fn copy_self(dest: &str) -> Result<()> { - let exe = std::env::current_exe().into_diagnostic()?; - - let dest_path = Path::new(dest); - let final_path = if dest_path.is_dir() { - let file_name = exe - .file_name() - .ok_or_else(|| miette::miette!("current_exe has no file name: {}", exe.display()))?; - dest_path.join(file_name) - } else { - dest_path.to_path_buf() - }; - - if let Some(parent) = final_path.parent() - && !parent.as_os_str().is_empty() - { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - - std::fs::copy(&exe, &final_path).into_diagnostic()?; - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let mut perms = std::fs::metadata(&final_path) - .into_diagnostic()? - .permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&final_path, perms).into_diagnostic()?; - } - - Ok(()) -} - -#[cfg(target_os = "linux")] -fn prepare_sidecar_directory(path: &Path, uid: u32, gid: u32, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; - chown(path, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {uid}:{gid}", - path.display() - ) - })?; - Ok(()) -} - -#[cfg(target_os = "linux")] -fn prepare_sidecar_directory_for_current_user(path: &Path, mode: u32) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - let uid = Uid::current(); - let gid = Gid::current(); - std::fs::create_dir_all(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to create sidecar directory {}", path.display()))?; - chown(path, Some(uid), Some(gid)) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown sidecar directory {} to {}:{}", - path.display(), - uid.as_raw(), - gid.as_raw() - ) - })?; - let mut perms = std::fs::metadata(path).into_diagnostic()?.permissions(); - perms.set_mode(mode); - std::fs::set_permissions(path, perms) - .into_diagnostic() - .wrap_err_with(|| format!("failed to chmod sidecar directory {}", path.display()))?; - Ok(()) -} - -#[cfg(target_os = "linux")] -fn copy_sidecar_client_tls_if_present( - source_dir: &Path, - sidecar_tls_dir: &Path, - uid: u32, - gid: u32, -) -> Result<()> { - use miette::Context as _; - use nix::unistd::{Gid, Uid, chown}; - use std::os::unix::fs::PermissionsExt; - - if !source_dir.exists() { - return Ok(()); - } - - let dest_dir = sidecar_tls_dir.join(SIDECAR_CLIENT_TLS_SUBDIR); - prepare_sidecar_directory_for_current_user(&dest_dir, SIDECAR_TLS_STAGING_DIR_MODE)?; - for file_name in CLIENT_TLS_FILES { - let source = source_dir.join(file_name); - if !source.exists() { - return Err(miette::miette!( - "client TLS source file is missing: {}", - source.display() - )); - } - let dest = dest_dir.join(file_name); - if dest.exists() { - std::fs::remove_file(&dest) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to remove stale client TLS file {}", dest.display()) - })?; - } - std::fs::copy(&source, &dest) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to copy client TLS file {} to {}", - source.display(), - dest.display() - ) - })?; - let mut perms = std::fs::metadata(&dest).into_diagnostic()?.permissions(); - perms.set_mode(SIDECAR_CLIENT_TLS_FILE_MODE); - std::fs::set_permissions(&dest, perms) - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to chmod copied client TLS file {}", dest.display()) - })?; - chown(&dest, Some(Uid::from_raw(uid)), Some(Gid::from_raw(gid))) - .into_diagnostic() - .wrap_err_with(|| { - format!( - "failed to chown copied client TLS file {} to {uid}:{gid}", - dest.display() - ) - })?; - } - - prepare_sidecar_directory(&dest_dir, uid, gid, SIDECAR_CLIENT_TLS_DIR_MODE)?; - - Ok(()) -} - -#[cfg(target_os = "linux")] -fn run_network_init( - proxy_user_id: u32, - proxy_primary_group_id: u32, - sidecar_state_dir: &str, - sidecar_tls_dir: &str, -) -> Result<()> { - validate_network_init_ids(proxy_user_id, proxy_primary_group_id)?; - - let sidecar_state_dir = Path::new(sidecar_state_dir); - let sidecar_tls_dir = Path::new(sidecar_tls_dir); - prepare_sidecar_directory( - sidecar_state_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_STATE_DIR_MODE, - )?; - // The init container runs as uid 0 with CAP_DAC_OVERRIDE dropped. Keep the - // TLS work directory owned by the init user until the client cert copy is - // complete, then hand it to the long-running proxy UID. - prepare_sidecar_directory_for_current_user(sidecar_tls_dir, SIDECAR_TLS_DIR_MODE)?; - copy_sidecar_client_tls_if_present( - Path::new(CLIENT_TLS_DIR), - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - )?; - prepare_sidecar_directory( - sidecar_tls_dir, - proxy_user_id, - proxy_primary_group_id, - SIDECAR_TLS_DIR_MODE, - )?; - openshell_supervisor_process::netns::install_sidecar_bypass_rules(proxy_user_id) -} - -#[cfg(target_os = "linux")] -fn validate_network_init_ids(proxy_user_id: u32, proxy_primary_group_id: u32) -> Result<()> { - if proxy_user_id != 0 && proxy_user_id < openshell_policy::MIN_SANDBOX_UID { - return Err(miette::miette!( - "--proxy-uid must be 0 or at least {}", - openshell_policy::MIN_SANDBOX_UID - )); - } - if proxy_primary_group_id < openshell_policy::MIN_SANDBOX_UID { - return Err(miette::miette!( - "--proxy-gid must be at least {}", - openshell_policy::MIN_SANDBOX_UID - )); - } - Ok(()) -} - -#[cfg(not(target_os = "linux"))] -fn run_network_init( - _proxy_uid: u32, - _proxy_gid: u32, - _sidecar_state_dir: &str, - _sidecar_tls_dir: &str, -) -> Result<()> { - Err(miette::miette!( - "--mode=network-init is only supported on Linux" - )) -} - -fn main() -> Result<()> { - // Handle `copy-self ` before clap so it works without any of the - // sandbox flags. Kubernetes init containers invoke this path to seed an - // emptyDir volume that the agent container then executes from. - let raw_args: Vec = std::env::args().collect(); - if raw_args.get(1).map(String::as_str) == Some(COPY_SELF_SUBCOMMAND) { - let dest = raw_args.get(2).ok_or_else(|| { - miette::miette!("usage: openshell-sandbox {COPY_SELF_SUBCOMMAND} ") - })?; - return copy_self(dest); - } - - // Handle `debug-rpc [args]` before clap. Uses a small - // dedicated runtime so we don't pay the supervisor's full startup cost. - if raw_args.get(1).map(String::as_str) == Some(DEBUG_RPC_SUBCOMMAND) { - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .into_diagnostic()?; - return runtime.block_on(async move { - let _ = rustls::crypto::ring::default_provider().install_default(); - let exit = openshell_supervisor_process::debug_rpc::run(&raw_args[2..]).await?; - std::process::exit(exit); - }); - } - if raw_args.get(1).map(String::as_str) == Some(VALIDATE_WORKSPACE_SUBCOMMAND) { - return validate_workspace(&raw_args[2..]); - } - - let args = Args::parse(); - - if args.mode.network_init { - let proxy_gid = args.proxy_gid.unwrap_or(args.proxy_uid); - return run_network_init( - args.proxy_uid, - proxy_gid, - &args.sidecar_state_dir, - &args.sidecar_tls_dir, - ); - } - - // Try to open a rolling log file; fall back to stderr-only logging if it fails - // (e.g., /var/log is not writable in custom workload images). - // Rotates daily, keeps the 3 most recent files to bound disk usage. - let file_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - (writer, guard) - }); - - let console_filter = - EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(&args.log_level)); - - let runtime = tokio::runtime::Builder::new_multi_thread() - .enable_all() - .build() - .into_diagnostic()?; - - let exit_code = runtime.block_on(async move { - // Install rustls crypto provider before any TLS connections (including log push). - let _ = rustls::crypto::ring::default_provider().install_default(); - - // Set up optional log push layer (gRPC mode only). - let log_push_state = if let (Some(sandbox_id), Some(endpoint)) = - (&args.sandbox_id, &args.openshell_endpoint) - { - let (tx, handle) = openshell_supervisor_process::log_push::spawn_log_push_task( - endpoint.clone(), - sandbox_id.clone(), - ); - let layer = - openshell_supervisor_process::log_push::LogPushLayer::new(sandbox_id.clone(), tx); - Some((layer, handle)) - } else { - None - }; - let push_layer = log_push_state.as_ref().map(|(layer, _)| layer.clone()); - let _log_push_handle = log_push_state.map(|(_, handle)| handle); - - // Shared flag: the sandbox poll loop toggles this when the - // `ocsf_json_enabled` setting changes. The JSONL layer checks it - // on each event and short-circuits when false. - let ocsf_enabled = Arc::new(AtomicBool::new(false)); - - // Keep guards alive for the entire process. When a guard is dropped the - // non-blocking writer flushes remaining logs. - let (_file_guard, _jsonl_guard) = if let Some((file_writer, file_guard)) = file_logging { - let file_filter = EnvFilter::new("info"); - - // OCSF JSONL file: rolling appender matching the main log file - // (daily rotation, 3 files max). Created eagerly but gated by the - // enabled flag — no JSONL is written until ocsf_json_enabled is set. - let jsonl_logging = tracing_appender::rolling::RollingFileAppender::builder() - .rotation(tracing_appender::rolling::Rotation::DAILY) - .filename_prefix("openshell-ocsf") - .filename_suffix("log") - .max_log_files(3) - .build("/var/log") - .ok() - .map(|roller| { - let (writer, guard) = tracing_appender::non_blocking(roller); - let layer = OcsfJsonlLayer::new(writer).with_enabled_flag(ocsf_enabled.clone()); - (layer, guard) - }); - let (jsonl_layer, jsonl_guard) = match jsonl_logging { - Some((layer, guard)) => (Some(layer), Some(guard)), - None => (None, None), - }; - - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with( - OcsfShorthandLayer::new(file_writer) - .with_non_ocsf(true) - .with_filter(file_filter), - ) - .with(jsonl_layer.with_filter(LevelFilter::INFO)) - .with(push_layer.clone()) - .init(); - (Some(file_guard), jsonl_guard) - } else { - tracing_subscriber::registry() - .with( - OcsfShorthandLayer::new(std::io::stderr()) - .with_non_ocsf(true) - .with_filter(console_filter), - ) - .with(push_layer) - .init(); - // Log the warning after the subscriber is initialized - warn!("Could not open /var/log for log rotation; using stderr-only logging"); - (None, None) - }; - - // Get command - either from CLI args, environment variable, or default to /bin/bash - let command = if !args.command.is_empty() { - args.command - } else if let Ok(c) = std::env::var(openshell_core::sandbox_env::SANDBOX_COMMAND) { - // Simple shell-like splitting on whitespace - c.split_whitespace().map(String::from).collect() - } else { - vec!["/bin/bash".to_string()] - }; - - info!(command = ?command, "Starting sandbox"); - // Note: "Starting sandbox" stays as plain info!() since the OCSF context - // is not yet initialized at this point (run_sandbox hasn't been called). - // The shorthand layer will render it in fallback format. - - let upstream_proxy_args = openshell_supervisor_network::upstream_proxy::UpstreamProxyArgs { - https_proxy: args.upstream_proxy, - no_proxy: args.upstream_no_proxy, - proxy_auth_file: args.upstream_proxy_auth_file, - proxy_auth_allow_insecure: args.upstream_proxy_auth_allow_insecure, - proxy_connect_by_hostname: args.upstream_proxy_connect_by_hostname, - }; - - run_sandbox( - command, - args.workdir, - args.timeout, - args.interactive, - args.sandbox_id, - args.sandbox, - args.openshell_endpoint, - args.policy_rules, - args.policy_data, - args.ssh_socket_path, - args.health_check, - args.health_port, - args.inference_routes, - ocsf_enabled, - args.mode.network, - args.mode.process, - upstream_proxy_args, - ) - .await - })?; - - std::process::exit(exit_code); -} - -#[cfg(test)] -mod tests { - use super::*; - use std::os::unix::fs::PermissionsExt; - - #[cfg(target_os = "linux")] - #[test] - fn workspace_validation_subcommand_uses_final_policy_identity() { - let uid = nix::unistd::geteuid().as_raw(); - let gid = nix::unistd::getegid().as_raw(); - if uid < 1000 || gid < 1000 { - return; - } - let dir = tempfile::tempdir_in("/tmp").unwrap(); - std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o711)).unwrap(); - let root = dir.path().canonicalize().unwrap().join("workspace"); - std::fs::create_dir(&root).unwrap(); - std::fs::set_permissions(&root, std::fs::Permissions::from_mode(0o700)).unwrap(); - let args = vec![ - "--workdir".to_string(), - root.display().to_string(), - "--expected-uid".to_string(), - uid.to_string(), - "--expected-gid".to_string(), - gid.to_string(), - ]; - - validate_workspace(&args).expect("current identity should retain workspace authority"); - } - - /// Drives `copy_self`'s file-copy logic against an arbitrary source path - /// so tests don't depend on `current_exe()`. - fn copy_executable(src: &Path, dest: &Path) -> Result<()> { - let final_path = if dest.is_dir() { - dest.join(src.file_name().unwrap()) - } else { - dest.to_path_buf() - }; - if let Some(parent) = final_path.parent() - && !parent.as_os_str().is_empty() - { - std::fs::create_dir_all(parent).into_diagnostic()?; - } - std::fs::copy(src, &final_path).into_diagnostic()?; - let mut perms = std::fs::metadata(&final_path) - .into_diagnostic()? - .permissions(); - perms.set_mode(0o755); - std::fs::set_permissions(&final_path, perms).into_diagnostic()?; - Ok(()) - } - - #[test] - fn copy_self_writes_executable_at_target_path() { - let tmp = tempfile::tempdir().unwrap(); - let src = tmp.path().join("source-bin"); - std::fs::write(&src, b"#!/bin/false\n").unwrap(); - - let dest = tmp.path().join("subdir/openshell-sandbox"); - copy_executable(&src, &dest).unwrap(); - - assert!(dest.exists(), "destination file should exist"); - let mode = std::fs::metadata(&dest).unwrap().permissions().mode(); - assert_eq!(mode & 0o777, 0o755, "destination must be 0755"); - let copied = std::fs::read(&dest).unwrap(); - assert_eq!(copied, b"#!/bin/false\n"); - } - - #[test] - fn copy_self_into_existing_directory_uses_source_filename() { - let tmp = tempfile::tempdir().unwrap(); - let src = tmp.path().join("openshell-sandbox"); - std::fs::write(&src, b"binary").unwrap(); - - let dest_dir = tmp.path().join("bin"); - std::fs::create_dir_all(&dest_dir).unwrap(); - - copy_executable(&src, &dest_dir).unwrap(); - - let final_path = dest_dir.join("openshell-sandbox"); - assert!(final_path.exists(), "binary should land inside dest dir"); - } - - #[test] - fn mode_parses_network_init_standalone() { - let mode = "network-init".parse::().unwrap(); - assert!(mode.network_init); - assert!(!mode.network); - assert!(!mode.process); - } - - #[test] - fn mode_rejects_combined_network_init() { - let err = "network-init,network".parse::().unwrap_err(); - assert!(err.contains("cannot be combined")); - } - - #[test] - fn mode_rejects_empty_value() { - let err = "".parse::().unwrap_err(); - assert!(err.contains("at least one")); - } - - #[cfg(target_os = "linux")] - #[test] - fn sidecar_tls_modes_preserve_proxy_owned_parent_and_private_client_dir() { - assert_eq!(SIDECAR_TLS_DIR_MODE, 0o755); - assert_eq!(SIDECAR_TLS_STAGING_DIR_MODE, 0o700); - assert_eq!(SIDECAR_CLIENT_TLS_DIR_MODE, 0o750); - assert_eq!(SIDECAR_CLIENT_TLS_FILE_MODE, 0o400); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_accepts_root_proxy_uid_for_binary_aware_sidecar() { - validate_network_init_ids(0, openshell_policy::MIN_SANDBOX_UID).unwrap(); - } - - #[cfg(target_os = "linux")] - #[test] - fn network_init_still_rejects_low_non_root_proxy_ids() { - let uid_err = - validate_network_init_ids(999, openshell_policy::MIN_SANDBOX_UID).unwrap_err(); - assert!(uid_err.to_string().contains("--proxy-uid")); - - let gid_err = validate_network_init_ids(0, 999).unwrap_err(); - assert!(gid_err.to_string().contains("--proxy-gid")); - } -} diff --git a/crates/openshell-sandbox/src/main_win.rs b/crates/openshell-sandbox/src/main_win.rs new file mode 100644 index 000000000..460b379c4 --- /dev/null +++ b/crates/openshell-sandbox/src/main_win.rs @@ -0,0 +1,7 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +fn main() { + eprintln!("openshell-sandbox supervisor is not available on Windows"); + std::process::exit(1); +}