diff --git a/architecture/sandbox.md b/architecture/sandbox.md index 7368ca905..dd71239cd 100644 --- a/architecture/sandbox.md +++ b/architecture/sandbox.md @@ -52,18 +52,29 @@ paths, such as proxy support files or GPU device paths when a GPU is present. ## Isolation Backend The supervisor does not build the boundary inline. It drives an **isolation -backend** (RFC 0012) through one runtime contract (`ensure_boundary` → `bind` -→ `boundary_ready` → `start_agent`), so the same supervisor code runs wherever -the boundary sits. Today only the in-pod backend exists: it builds the boundary -in the same process that operates it, over the existing primitives (network -namespace, proxy, the pre-exec Landlock/seccomp ceiling, procfs identity). +backend** (RFC 0012) through one runtime contract (`attach` → `confirm` → +`start_agent`, over the `Bound`/`Ready`/`Running` type-state), so the same +supervisor code runs wherever the boundary sits. The supervisor selects the +backend from the driver-provided `TopologyDescriptor` and never branches on +which one it resolved. Two backends exist: -The contract covers the full isolation envelope. `ensure_boundary` establishes -the network namespace; `start_agent` establishes filesystem (Landlock) and -syscall (seccomp) at process entry; `identity` resolves the connecting binary -from procfs at runtime. The lifecycle handles are unforgeable type-state -tokens, so the order (and thus "no workload runs before the boundary is -ready") holds by construction rather than by convention. +- **in-pod** — builds and operates the boundary in the same process, over the + existing primitives (network namespace, proxy, the pre-exec Landlock/seccomp + ceiling, procfs identity). +- **sidecar** (Kubernetes) — the agent container's supervisor runs this backend + to operate the workload; network mediation runs in a separate network-sidecar + container the backend coordinates (the agent container holds no gateway + credentials and receives its policy over a peer-cred-authenticated control + socket). The agent stays autonomous — it spawns its own workload — so the + backend wraps that path rather than inverting control. + +`attach` binds the trusted sandbox context and establishes standing enforcement; +`confirm` verifies it (fail-closed); `start_agent` establishes the per-process +launch-time controls (Landlock/seccomp) before the first untrusted instruction; +identity resolves the connecting binary from procfs at runtime. The lifecycle +handles are unforgeable type-state tokens, so the order (and thus "no workload +runs before the boundary is ready") holds by construction rather than by +convention. ## Network and Inference diff --git a/crates/openshell-driver-kubernetes/src/driver.rs b/crates/openshell-driver-kubernetes/src/driver.rs index 9fc0509d3..b900f2314 100644 --- a/crates/openshell-driver-kubernetes/src/driver.rs +++ b/crates/openshell-driver-kubernetes/src/driver.rs @@ -1996,6 +1996,16 @@ fn apply_supervisor_sidecar_topology( openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET, SIDECAR_CONTROL_SOCKET, ); + // RFC 0012: the agent container's supervisor resolves the sidecar + // backend, so override the default in-pod descriptor set by + // `apply_required_env`. (The network sidecar container keeps the + // in-pod default; it runs the mediation component, not a backend, so + // its descriptor is never resolved.) + upsert_env( + env, + openshell_core::sandbox_env::TOPOLOGY_DESCRIPTOR, + &openshell_isolation::contract::TopologyDescriptor::sidecar().to_env_value(), + ); upsert_env( env, openshell_core::sandbox_env::PROXY_TLS_DIR, @@ -4255,6 +4265,13 @@ mod tests { rendered_env(agent, openshell_core::sandbox_env::SIDECAR_CONTROL_SOCKET), Some(SIDECAR_CONTROL_SOCKET) ); + // RFC 0012: the agent container resolves the sidecar backend. + let sidecar_descriptor = + openshell_isolation::contract::TopologyDescriptor::sidecar().to_env_value(); + assert_eq!( + rendered_env(agent, openshell_core::sandbox_env::TOPOLOGY_DESCRIPTOR), + Some(sidecar_descriptor.as_str()) + ); assert_eq!(rendered_env(agent, "OPENSHELL_SUPERVISOR_READY_FILE"), None); assert_eq!(rendered_env(agent, "OPENSHELL_ENTRYPOINT_PID_FILE"), None); assert_eq!( @@ -4288,6 +4305,16 @@ mod tests { sidecar["command"], serde_json::json!([SUPERVISOR_IMAGE_BINARY_PATH, "--mode=network"]) ); + // The network sidecar container runs the mediation component, not a + // backend, so it keeps the default in-pod descriptor (never resolved). + assert_eq!( + rendered_env(sidecar, openshell_core::sandbox_env::TOPOLOGY_DESCRIPTOR), + Some( + openshell_isolation::contract::TopologyDescriptor::in_pod() + .to_env_value() + .as_str() + ) + ); assert_eq!(sidecar["securityContext"]["runAsUser"], 0); assert_eq!(sidecar["securityContext"]["runAsGroup"], 1500); assert_eq!(sidecar["securityContext"]["runAsNonRoot"], false); diff --git a/crates/openshell-isolation/src/contract.rs b/crates/openshell-isolation/src/contract.rs index a7a59c57e..201b724a1 100644 --- a/crates/openshell-isolation/src/contract.rs +++ b/crates/openshell-isolation/src/contract.rs @@ -52,6 +52,16 @@ pub const CONTRACT_VERSION: u32 = 1; /// `openshell-sandbox`. pub const IN_POD_BACKEND_ID: &str = "in-pod"; +/// The well-known backend id of the Kubernetes sidecar topology. +/// +/// Selected for the agent container, whose supervisor runs this backend to +/// operate the workload; network mediation runs in a separate network-sidecar +/// container that the backend coordinates. A distinct topology from `in-pod`. +/// Lives in the contract crate because compute drivers create the +/// [`TopologyDescriptor`] naming it; the implementation lives in +/// `openshell-sandbox`. +pub const SIDECAR_BACKEND_ID: &str = "sidecar"; + // ============================================================================ // Errors // ============================================================================ @@ -190,6 +200,19 @@ impl TopologyDescriptor { } } + /// The descriptor for the Kubernetes sidecar topology's agent container: the + /// supervisor there resolves the sidecar backend. The payload is empty; the + /// backend reads its operational data (control socket, expected peer) from + /// the driver-controlled environment for now. + #[must_use] + pub fn sidecar() -> Self { + Self { + contract_version: CONTRACT_VERSION, + backend_id: SIDECAR_BACKEND_ID.to_string(), + payload: Vec::new(), + } + } + /// Serialize for the driver-controlled environment transport /// (`OPENSHELL_TOPOLOGY_DESCRIPTOR`): `::`. /// Other transports deliver the same envelope with equivalent integrity. @@ -251,7 +274,7 @@ fn hex_encode(bytes: &[u8]) -> String { } fn hex_decode(hex: &str) -> Option> { - if !hex.is_ascii() || hex.len() % 2 != 0 { + if !hex.is_ascii() || !hex.len().is_multiple_of(2) { return None; } (0..hex.len()) @@ -444,9 +467,11 @@ pub trait BoundBoundary: Send { async fn confirm(self: Box) -> Result, BackendError>; } -/// Ready: mediation is initialized, standing enforcement is confirmed, and the -/// backend is prepared to ensure the admitted launch-time controls are in force -/// before untrusted execution. Only agent activation is possible from here. +/// Ready: mediation is initialized and standing enforcement is confirmed. +/// +/// The backend is prepared to ensure the admitted launch-time controls are in +/// force before untrusted execution. Only agent activation is possible from +/// here. #[async_trait] pub trait ReadyBoundary: Send { /// Make the admitted agent runnable behind the boundary and return its @@ -459,10 +484,11 @@ pub trait ReadyBoundary: Send { async fn start_agent(self: Box) -> Result, BackendError>; } -/// Running: the agent is runnable behind the boundary and the returned agent -/// handle owns the complete workload tree. Exec and forwarding are available. -/// All interface accessors return owned `Arc`s so a consumer can retain them -/// past any later state consumption. +/// Running: the agent is runnable behind the boundary. +/// +/// The returned agent handle owns the complete workload tree. Exec and +/// forwarding are available. All interface accessors return owned `Arc`s so a +/// consumer can retain them past any later state consumption. pub trait RunningBoundary: Send + Sync { /// The agent process handle (owns the complete workload tree). fn agent(&self) -> Arc; @@ -558,6 +584,7 @@ pub struct ExecSpec { } /// In-boundary process entry, consumed by the SSH server and supervisor session. +/// /// Like `start_agent`, every exec ensures the applicable launch-time controls /// are in force before the new process executes its first untrusted instruction /// and preserves the provisioned execution environment. diff --git a/crates/openshell-isolation/src/lib.rs b/crates/openshell-isolation/src/lib.rs index b900058c8..d0ebd8831 100644 --- a/crates/openshell-isolation/src/lib.rs +++ b/crates/openshell-isolation/src/lib.rs @@ -26,7 +26,7 @@ //! # Ordering is a security property //! //! The lifecycle states run in order: attach -> Bound -> confirm -> Ready -> -//! start_agent -> Running. Nothing untrusted runs inside the boundary until it +//! `start_agent` -> Running. Nothing untrusted runs inside the boundary until it //! is confirmed ready. This is enforced *by construction*: each transition //! consumes the prior state by value, and no state type has a public //! constructor, so the supervisor cannot skip a stage or run a workload before diff --git a/crates/openshell-sandbox/src/lib.rs b/crates/openshell-sandbox/src/lib.rs index cdb6c26fb..2eef20eb4 100644 --- a/crates/openshell-sandbox/src/lib.rs +++ b/crates/openshell-sandbox/src/lib.rs @@ -13,6 +13,7 @@ mod inpod; mod mechanistic_mapper; #[cfg_attr(not(target_os = "linux"), allow(dead_code))] mod metadata_server; +mod sidecar; mod sidecar_control; use miette::{IntoDiagnostic, Result, WrapErr}; @@ -137,6 +138,20 @@ pub async fn run_sandbox( let process_enforcement_mode = process_enforcement_mode(); let process_uses_sidecar_control = process_enabled && !network_enabled && sidecar_network_enforcement; + + // Sidecar network enforcement requires the network and process roles to run + // in *separate* containers: the network container hosts egress mediation and + // the agent container runs the workload behind the sidecar backend. A single + // supervisor enabling both roles at once (the default `--mode=network,process` + // run by hand, plus sidecar enforcement) is not a topology any compute driver + // produces. Fail closed rather than silently entering the network-mediation + // arm and holding open without ever spawning the workload. + if sidecar_network_enforcement && network_enabled && process_enabled { + return Err(miette::miette!( + "sidecar network enforcement requires split containers: a single supervisor \ + cannot enable both --mode=network and --mode=process" + )); + } let mut process_control_connection = None; let sidecar_bootstrap = if process_uses_sidecar_control { let socket = sidecar_control_socket().ok_or_else(|| { @@ -434,7 +449,14 @@ pub async fn run_sandbox( // over the control socket in the workload container. Otherwise drive the // co-located in-pod isolation backend (RFC 0012), which owns the network // namespace, proxy, and workload behind the boundary lifecycle. - let exit_code = if sidecar_network_enforcement { + let exit_code = if sidecar_network_enforcement && network_enabled { + // The network-mediation component: this is the Kubernetes *network* + // container. It hosts the proxy (egress mediation) and the sidecar + // control server for the agent container's supervisor; it runs no + // workload of its own (`process_enabled` is false here). The agent + // container's process path is a separate RFC 0012 backend (see the + // `else` arm and `sidecar.rs`). + // // Legacy in-pod netns is never created under sidecar enforcement; the // expression evaluates to None here but is kept for structural parity // with the non-sidecar supervisor path. @@ -641,111 +663,16 @@ pub async fn run_sandbox( } } - let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; - let sidecar_bootstrap_ca_file_paths = sidecar_bootstrap.as_ref().and_then(|bootstrap| { - bootstrap - .proxy_ca_cert_path - .clone() - .zip(bootstrap.proxy_ca_bundle_path.clone()) - }); - - let exit_code = if process_enabled { - let ca_file_paths = networking - .as_ref() - .and_then(|n| n.ca_file_paths.clone()) - .or_else(|| { - if sidecar_network_enforcement { - sidecar_bootstrap_ca_file_paths - .clone() - .or_else(sidecar_ca_file_paths) - } else { - None - } - }); - - let entrypoint_started_tx = if process_uses_sidecar_control - && let Some(writer) = process_control_writer.clone() - { - let (tx, rx) = tokio::sync::oneshot::channel(); - tokio::spawn(async move { - match rx.await { - Ok(pid) => { - if let Err(err) = - sidecar_control::send_entrypoint_started(&writer, pid).await - { - warn!(error = %err, "Failed to send sidecar entrypoint event"); - } - } - Err(_closed) => { - debug!("Entrypoint exited before sidecar entrypoint event was sent"); - } - } - }); - Some(tx) - } else { - None - }; - - // The workload spawn returns an owned handle (RFC 0012); wait on it - // to obtain the exit code, racing the authoritative sidecar control - // channel so a supervisor disconnect terminates the workload. - let mut agent = openshell_supervisor_process::run::spawn_workload( - program, - args, - workdir.as_deref(), - timeout_secs, - interactive, - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - ssh_socket_path, - sidecar_network_enforcement, - &process_policy, - process_enforcement_mode, - entrypoint_pid, - entrypoint_started_tx, - provider_credentials, - provider_env, - ca_file_paths, - #[cfg(target_os = "linux")] - netns.as_ref(), - #[cfg(target_os = "linux")] - bypass_denial_tx, - #[cfg(target_os = "linux")] - bypass_activity_tx, - ) - .await?; - - if let Some(control_closed) = process_control_closed.as_mut() { - tokio::select! { - result = agent.wait() => result?, - _ = control_closed => { - ocsf_emit!( - AppLifecycleBuilder::new(ocsf_ctx()) - .activity(ActivityId::Fail) - .severity(SeverityId::High) - .status(StatusId::Failure) - .message( - "Authoritative network-sidecar control channel closed; terminating process container" - ) - .build() - ); - return Err(miette::miette!( - "authoritative network-sidecar control channel closed" - )); - } - } - } else { - agent.wait().await? - } - } else { - // Network-only sidecar mode: keep the proxy and its background - // tasks alive (held via the `networking` value) until shutdown. If - // the sole authenticated process-supervisor control connection - // closes, exit non-zero so Kubernetes restarts the network sidecar - // and creates a fresh one-client bootstrap listener for the - // restarted agent. + // Network-only sidecar mode: this network container keeps the proxy and + // its background tasks alive (held via the `networking` value) until + // shutdown. The agent container's workload is spawned by the sidecar + // isolation backend (see the `else` arm); it never runs here. If the + // sole authenticated process-supervisor control connection closes, exit + // non-zero so Kubernetes restarts the network sidecar and creates a + // fresh one-client bootstrap listener for the restarted agent. + let exit_code = { #[cfg(target_os = "linux")] - if let Some(control_task) = sidecar_control_task { + let code = if let Some(control_task) = sidecar_control_task { tokio::select! { () = wait_for_shutdown_signal() => 0, result = control_task => { @@ -756,12 +683,13 @@ pub async fn run_sandbox( } else { wait_for_shutdown_signal().await; 0 - } + }; #[cfg(not(target_os = "linux"))] - { + let code = { wait_for_shutdown_signal().await; 0 - } + }; + code }; // Drop networking explicitly so the proxy + bypass monitor RAII handles @@ -778,37 +706,86 @@ pub async fn run_sandbox( // ready" is enforced by construction. let policy_local_slot = Arc::new(std::sync::Mutex::new(None)); - let config = inpod::InPodConfig { - network_enabled, - process_enabled, - opa_engine: opa_engine.clone(), - retained_proto, - entrypoint_pid: entrypoint_pid.clone(), - provider_credentials: provider_credentials.clone(), - provider_env: std::sync::Mutex::new(provider_env), - process_enforcement_mode, - sandbox_name: sandbox_name_for_agg.clone(), - openshell_endpoint: openshell_endpoint_for_proxy.clone(), - inference_routes, - ssh_socket_path, - workspace_rx: workspace_rx.clone(), - denial_tx: std::sync::Mutex::new(denial_tx), - activity_tx: std::sync::Mutex::new(activity_tx), - #[cfg(target_os = "linux")] - bypass_denial_tx: std::sync::Mutex::new(bypass_denial_tx), - #[cfg(target_os = "linux")] - bypass_activity_tx: std::sync::Mutex::new(bypass_activity_tx), - policy_local_slot: policy_local_slot.clone(), + // Build the admitted backend. The Kubernetes sidecar *agent* container + // (network mediation is enforced by a separate network container, so + // `sidecar_network_enforcement` is set but this container has + // `network_enabled == false`) drives the sidecar backend: it wraps the + // autonomous agent/process path (workload spawn + control-closed + // watchdog) in the RFC 0012 contract. Every other supervisor drives the + // co-located in-pod backend, which owns the netns, proxy, and workload. + let (factory, admitted_backend_id): ( + Arc, + &str, + ) = if sidecar_network_enforcement { + // Process-role policy and CA material for the agent container. The + // proxy runs in the network container, so there is no local + // `networking` to source CA paths from; fall back to the sidecar + // bootstrap or the well-known mount. + let process_policy = process_policy_for_topology(&policy, sidecar_network_enforcement)?; + let ca_file_paths = sidecar_bootstrap + .as_ref() + .and_then(|bootstrap| { + bootstrap + .proxy_ca_cert_path + .clone() + .zip(bootstrap.proxy_ca_bundle_path.clone()) + }) + .or_else(sidecar_ca_file_paths); + + let config = sidecar::SidecarConfig { + process_enforcement_mode, + provider_credentials: provider_credentials.clone(), + provider_env, + process_policy, + ca_file_paths, + entrypoint_pid: entrypoint_pid.clone(), + ssh_socket_path, + openshell_endpoint: openshell_endpoint_for_proxy.clone(), + sandbox_id: sandbox_id.clone(), + process_control_writer, + process_control_closed: std::sync::Mutex::new(process_control_closed), + #[cfg(target_os = "linux")] + bypass_denial_tx: std::sync::Mutex::new(bypass_denial_tx), + #[cfg(target_os = "linux")] + bypass_activity_tx: std::sync::Mutex::new(bypass_activity_tx), + }; + ( + Arc::new(sidecar::SidecarBackendFactory::new(config)), + sidecar::SIDECAR_BACKEND_ID, + ) + } else { + let config = inpod::InPodConfig { + network_enabled, + process_enabled, + opa_engine: opa_engine.clone(), + retained_proto, + entrypoint_pid: entrypoint_pid.clone(), + provider_credentials: provider_credentials.clone(), + provider_env: std::sync::Mutex::new(provider_env), + process_enforcement_mode, + sandbox_name: sandbox_name_for_agg.clone(), + openshell_endpoint: openshell_endpoint_for_proxy.clone(), + inference_routes, + ssh_socket_path, + workspace_rx: workspace_rx.clone(), + denial_tx: std::sync::Mutex::new(denial_tx), + activity_tx: std::sync::Mutex::new(activity_tx), + #[cfg(target_os = "linux")] + bypass_denial_tx: std::sync::Mutex::new(bypass_denial_tx), + #[cfg(target_os = "linux")] + bypass_activity_tx: std::sync::Mutex::new(bypass_activity_tx), + policy_local_slot: policy_local_slot.clone(), + }; + ( + Arc::new(inpod::InPodBackendFactory::new(config)), + inpod::IN_POD_BACKEND_ID, + ) }; - // The admitted backend for this sandbox. In-pod today; a real deployment - // derives this id (and the descriptor payload) from trusted deployment - // configuration and admission. - let admitted_backend_id = inpod::IN_POD_BACKEND_ID; let mut registry = openshell_isolation::contract::BackendRegistry::new(); registry - .register(Arc::new(inpod::InPodBackendFactory::new(config))) - .map_err(|e| miette::miette!("failed to register in-pod backend: {e}"))?; + .register(factory) + .map_err(|e| miette::miette!("failed to register isolation backend: {e}"))?; // The topology descriptor. A driver-provisioned sandbox receives it // through the driver-controlled environment (`OPENSHELL_TOPOLOGY_DESCRIPTOR`) @@ -865,12 +842,16 @@ pub async fn run_sandbox( // Retain the boundary's mediation ingress before consuming `Bound`. let _mediation_ingress = bound.mediation_ingress(); - // Spawn background policy poll task (gRPC mode only). - if let (Some(id), Some(endpoint), Some(engine)) = ( - sandbox_id.as_deref(), - openshell_endpoint.as_deref(), - opa_engine.as_ref(), - ) { + // Spawn background policy poll task (gRPC mode only). Skipped for the + // sidecar agent container, whose policy arrives over the control socket + // rather than a direct gateway poll. + if !process_uses_sidecar_control + && let (Some(id), Some(endpoint), Some(engine)) = ( + sandbox_id.as_deref(), + openshell_endpoint.as_deref(), + opa_engine.as_ref(), + ) + { let poll_id = id.to_string(); let poll_endpoint = endpoint.to_string(); let poll_engine = engine.clone(); diff --git a/crates/openshell-sandbox/src/sidecar.rs b/crates/openshell-sandbox/src/sidecar.rs new file mode 100644 index 000000000..fc8a43d80 --- /dev/null +++ b/crates/openshell-sandbox/src/sidecar.rs @@ -0,0 +1,703 @@ +// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! The Kubernetes sidecar isolation backend (RFC 0012 runtime-selectable contract). +//! +//! This is the split placement: a separate *network* container hosts the +//! mediation service (the proxy) and the sidecar control server, while the +//! *agent* container runs the single logical supervisor plus this backend and +//! spawns the untrusted workload. Network mediation is remote (in the network +//! container), so this backend hosts no mediation ingress of its own; the agent +//! container is autonomous and self-provisions its workload, exactly as before. +//! +//! This backend wraps that existing agent/process path in the object-safe boxed +//! state chain (`attach -> Bound -> confirm -> Ready -> start_agent -> Running`) +//! without changing its behavior. Standing enforcement (network mediation) and +//! the control connection were established by the supervisor preamble before the +//! backend is attached, so `attach` and `confirm` are trivial here — there is no +//! local netns or proxy to bring up. `start_agent` replicates the workload spawn +//! (`spawn_workload`'s pre-exec ceiling applies filesystem/Landlock + syscall/ +//! seccomp launch-time controls before the first untrusted instruction), wiring +//! the entrypoint-started event over the control writer and racing the workload +//! against the authoritative control-closed signal so a supervisor disconnect +//! fails the boundary closed. + +use std::collections::HashMap; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::AtomicU32; + +use async_trait::async_trait; + +#[cfg(target_os = "linux")] +use openshell_core::activity::ActivitySender; +#[cfg(target_os = "linux")] +use openshell_core::denial::DenialEvent; +use openshell_core::policy::SandboxPolicy; +use openshell_core::provider_credentials::ProviderCredentialState; +use openshell_isolation::contract::{ + BackendError, BoundBoundary, BoundaryExec, BoundaryExitStatus, BoundaryPortForward, + BoundaryProcess, BoundarySignal, CONTRACT_VERSION, ExecSession, ExecSpec, + IsolationBackendFactory, MediatedConnection, MediationIngress, ReadyBoundary, RunningBoundary, + SandboxContext, VerifiedTopologyDescriptor, +}; +use openshell_ocsf::{ActivityId, AppLifecycleBuilder, SeverityId, StatusId, ocsf_emit}; +use openshell_supervisor_process::boundary_io::NetnsPortForward; +use openshell_supervisor_process::process::ProcessEnforcementMode; +use openshell_supervisor_process::run::{AgentSignaler, SpawnedAgent, spawn_workload}; +use tokio::net::unix::OwnedWriteHalf; +#[cfg(target_os = "linux")] +use tokio::sync::mpsc::UnboundedSender; +use tracing::{debug, warn}; + +/// The registered id for the sidecar backend (defined in the contract crate so +/// compute drivers can name it in the descriptors they create). +pub use openshell_isolation::contract::SIDECAR_BACKEND_ID; + +// ============================================================================ +// Config and factory +// ============================================================================ + +/// Runtime collaborators the sidecar agent/process lifecycle needs, captured +/// once when the factory is built. These are the pieces of the agent-container +/// process path that are *not* carried in the trusted [`SandboxContext`]: +/// launch-time enforcement inputs, provider material, the process-role policy +/// (already transformed by `process_policy_for_topology`), and the live sidecar +/// control connection established in the supervisor preamble. Move-once values +/// (the control-closed receiver and bypass senders) are held behind a +/// `Mutex>` so the `&self` state methods can take them exactly when +/// the matching transition fires. +pub struct SidecarConfig { + /// Process launch-time enforcement level (typically `NetworkOnly` in the + /// sidecar topology, since the network container owns egress mediation). + pub process_enforcement_mode: ProcessEnforcementMode, + pub provider_credentials: ProviderCredentialState, + /// Child environment for the agent, resolved by the supervisor preamble. + pub provider_env: HashMap, + /// The process-role policy, already transformed for the split topology by + /// `process_policy_for_topology`. + pub process_policy: SandboxPolicy, + /// Proxy CA material for the workload's trust store, resolved by the + /// supervisor preamble (sidecar bootstrap or the well-known mount). + pub ca_file_paths: Option<(PathBuf, PathBuf)>, + pub entrypoint_pid: Arc, + pub ssh_socket_path: Option, + pub openshell_endpoint: Option, + pub sandbox_id: Option, + /// The entrypoint-started writer half of the sidecar control connection. + /// Cloneable (`Arc`), so it is not a move-once slot. + pub process_control_writer: Option>>, + /// The authoritative sidecar control-closed signal (move-once): consumed by + /// `start_agent` and raced against the workload so a supervisor disconnect + /// fails the boundary closed. + pub process_control_closed: Mutex>>, + /// Bypass-monitor denial / activity senders (consumed by `start_agent`). + #[cfg(target_os = "linux")] + pub bypass_denial_tx: Mutex>>, + #[cfg(target_os = "linux")] + pub bypass_activity_tx: Mutex>, +} + +/// The factory for the sidecar backend. Holds the per-sandbox [`SidecarConfig`] +/// and hands it to the boundary on the single `attach`. +pub struct SidecarBackendFactory { + config: Mutex>, +} + +impl SidecarBackendFactory { + /// Build the factory from its per-sandbox runtime collaborators. + #[must_use] + pub fn new(config: SidecarConfig) -> Self { + Self { + config: Mutex::new(Some(config)), + } + } +} + +#[async_trait] +impl IsolationBackendFactory for SidecarBackendFactory { + fn backend_id(&self) -> &'static str { + SIDECAR_BACKEND_ID + } + + fn contract_version(&self) -> u32 { + CONTRACT_VERSION + } + + async fn attach( + &self, + descriptor: VerifiedTopologyDescriptor, + sandbox: SandboxContext, + ) -> Result, BackendError> { + // Validate the sidecar payload at the trust boundary: the agent-container + // supervisor *is* the resource, so the payload carries nothing. Reject a + // non-empty payload as a malformed/tampered descriptor before binding. + if !descriptor.payload().is_empty() { + return Err(BackendError::Descriptor( + "sidecar descriptor payload must be empty".to_string(), + )); + } + // `attach` never binds a resource that is already bound to an active + // boundary: the agent-container supervisor is this process, bindable + // exactly once. There is no fallible establishment step after the take + // (mediation and the control connection were brought up by the preamble), + // so a consumed slot always means "already bound". + let config = self + .config + .lock() + .expect("sidecar config lock") + .take() + .ok_or_else(|| { + BackendError::Denied( + "sidecar resource is already bound to an active boundary".to_string(), + ) + })?; + + // No local netns and no local proxy: mediation runs in the network + // container and the control connection is already established. The + // boundary is bound to the trusted sandbox context here. + let mediation_ingress: Arc = Arc::new(SidecarMediationIngress); + + Ok(Box::new(SidecarBound { + config, + sandbox, + mediation_ingress, + })) + } +} + +// ============================================================================ +// Lifecycle states +// ============================================================================ + +/// Bound: the descriptor and trusted sandbox context are bound to this agent +/// container's supervisor. Network mediation is hosted remotely (network +/// container), so the ingress here is an inert stub. No untrusted workload code +/// is running. +struct SidecarBound { + config: SidecarConfig, + sandbox: SandboxContext, + mediation_ingress: Arc, +} + +#[async_trait] +impl BoundBoundary for SidecarBound { + fn mediation_ingress(&self) -> Arc { + self.mediation_ingress.clone() + } + + async fn confirm(self: Box) -> Result, BackendError> { + // Standing enforcement (egress mediation) is confirmed by the network + // container that owns the proxy; the agent container cannot certify that + // remote proxy. It can, and must, certify the one signal it owns: the + // authoritative control link to the network container, established in the + // supervisor preamble. If that link is already down at confirmation time, + // fail closed rather than advance to `start_agent` and launch the + // untrusted workload only to tear it down on the first `wait`. + { + let mut guard = self + .config + .process_control_closed + .lock() + .expect("process_control_closed lock"); + if let Some(rx) = guard.as_mut() { + // The reader task sends `()` on close and drops the sender if it + // is aborted, so both `Ok(())` and `Closed` mean the link is + // gone; only `Empty` is still live. + match rx.try_recv() { + Ok(()) | Err(tokio::sync::oneshot::error::TryRecvError::Closed) => { + return Err(BackendError::Confirm( + "sidecar control channel already closed; refusing to start workload" + .to_string(), + )); + } + Err(tokio::sync::oneshot::error::TryRecvError::Empty) => {} + } + } + } + Ok(Box::new(SidecarReady { + config: self.config, + sandbox: self.sandbox, + })) + } +} + +/// Ready: standing enforcement is confirmed remotely and the control connection +/// is live. Only agent activation is possible. +struct SidecarReady { + config: SidecarConfig, + sandbox: SandboxContext, +} + +#[async_trait] +impl ReadyBoundary for SidecarReady { + async fn start_agent(self: Box) -> Result, BackendError> { + let this = *self; + let config = this.config; + let sandbox = this.sandbox; + + let port_forward: Arc = + Arc::new(NetnsPortForward { netns_fd: None }); + let exec: Arc = Arc::new(SidecarExec); + + let spec = &sandbox.agent; + + // Wire the entrypoint-started oneshot to the sidecar control writer: when + // the workload's entrypoint PID is known, forward it over the control + // connection so the network container can attribute the workload. + let entrypoint_started_tx = config.process_control_writer.clone().map(|writer| { + let (tx, rx) = tokio::sync::oneshot::channel(); + tokio::spawn(async move { + match rx.await { + Ok(pid) => { + if let Err(err) = + crate::sidecar_control::send_entrypoint_started(&writer, pid).await + { + warn!(error = %err, "Failed to send sidecar entrypoint event"); + } + } + Err(_closed) => { + debug!("Entrypoint exited before sidecar entrypoint event was sent"); + } + } + }); + tx + }); + + #[cfg(target_os = "linux")] + let bypass_denial_tx = config + .bypass_denial_tx + .lock() + .expect("bypass_denial_tx lock") + .take(); + #[cfg(target_os = "linux")] + let bypass_activity_tx = config + .bypass_activity_tx + .lock() + .expect("bypass_activity_tx lock") + .take(); + + // The sidecar agent container creates the workload itself; the + // launch-time controls (Landlock, seccomp, privilege drop as permitted + // by the enforcement mode) are applied inside `spawn_workload`'s pre-exec + // ceiling, before the first untrusted instruction. + let spawned = spawn_workload( + &spec.program, + &spec.args, + spec.workdir.as_deref(), + spec.timeout_secs, + spec.interactive, + config.sandbox_id.as_deref(), + config.openshell_endpoint.as_deref(), + config.ssh_socket_path.clone(), + // The agent container shares its SSH socket with the network sidecar + // container. + true, + &config.process_policy, + config.process_enforcement_mode, + config.entrypoint_pid.clone(), + entrypoint_started_tx, + config.provider_credentials.clone(), + config.provider_env.clone(), + config.ca_file_paths.clone(), + #[cfg(target_os = "linux")] + None, + #[cfg(target_os = "linux")] + bypass_denial_tx, + #[cfg(target_os = "linux")] + bypass_activity_tx, + ) + .await + .map_err(|e| BackendError::Process(e.to_string()))?; + + let control_closed = config + .process_control_closed + .lock() + .expect("process_control_closed lock") + .take(); + + let agent: Arc = + Arc::new(SidecarAgentProcess::running(spawned, control_closed)); + + Ok(Box::new(SidecarRunning { + agent, + exec, + port_forward, + })) + } +} + +/// Running: the workload is runnable behind the boundary. Exec, forwarding, +/// wait, and signal are available. The mediation ingress was retained by the +/// supervisor from the `Bound` state. +struct SidecarRunning { + agent: Arc, + exec: Arc, + port_forward: Arc, +} + +impl RunningBoundary for SidecarRunning { + fn agent(&self) -> Arc { + self.agent.clone() + } + fn exec(&self) -> Arc { + self.exec.clone() + } + fn port_forward(&self) -> Arc { + self.port_forward.clone() + } +} + +// ============================================================================ +// Agent process handle +// ============================================================================ + +/// The workload process running inside the sidecar agent container. `wait` +/// returns a stable terminal status across repeated calls and races the workload +/// against the authoritative control-closed signal: if the network container's +/// control connection closes, the boundary fails closed (`Exited(1)`). Signals +/// go through the lock-free pid-based [`AgentSignaler`] so they never contend +/// with an in-flight `wait`. +struct SidecarAgentProcess { + pid: Option, + signaler: Option, + waiter: tokio::sync::Mutex, +} + +enum AgentWaitState { + Pending { + agent: SpawnedAgent, + control_closed: Option>, + }, + Done(BoundaryExitStatus), +} + +impl SidecarAgentProcess { + fn running( + spawned: SpawnedAgent, + control_closed: Option>, + ) -> Self { + Self { + pid: Some(spawned.pid()), + signaler: Some(spawned.signaler()), + waiter: tokio::sync::Mutex::new(AgentWaitState::Pending { + agent: spawned, + control_closed, + }), + } + } +} + +#[async_trait] +impl BoundaryProcess for SidecarAgentProcess { + async fn wait(&self) -> Result { + // Holding the lock across the wait serializes repeated callers: the first + // performs the wait and caches the status; later callers block on the + // lock, then observe the cached `Done`. Signals never take this lock. + // + // `BackendError` is `!Clone`, so a fallible wait can never be cached as a + // `Result`; on any wait/spawn error the status is cached as `Exited(1)`. + let mut guard = self.waiter.lock().await; + match &mut *guard { + AgentWaitState::Done(status) => Ok(*status), + AgentWaitState::Pending { + agent, + control_closed, + } => { + let status = if let Some(closed) = control_closed.take() { + tokio::select! { + result = agent.wait() => wait_result_to_status(result), + _ = closed => { + // The authoritative network-sidecar control channel + // closed: fail the boundary closed. + ocsf_emit!( + AppLifecycleBuilder::new(crate::ocsf_ctx()) + .activity(ActivityId::Fail) + .severity(SeverityId::High) + .status(StatusId::Failure) + .message( + "Authoritative network-sidecar control channel closed; terminating agent container" + ) + .build() + ); + BoundaryExitStatus::Exited(1) + } + } + } else { + wait_result_to_status(agent.wait().await) + }; + *guard = AgentWaitState::Done(status); + Ok(status) + } + } + } + + async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> { + let Some(signaler) = self.signaler.as_ref() else { + return Ok(()); + }; + let result = match signal { + BoundarySignal::Term => signaler.term(), + BoundarySignal::Kill => signaler.kill(), + BoundarySignal::Int => signaler.interrupt(), + BoundarySignal::Hup => signaler.hangup(), + }; + result.map_err(|e| BackendError::Process(e.to_string())) + } + + async fn terminate(&self) -> Result<(), BackendError> { + let Some(signaler) = self.signaler.as_ref() else { + return Ok(()); + }; + signaler + .term() + .map_err(|e| BackendError::Process(e.to_string())) + } + + fn diagnostic_pid(&self) -> Option { + self.pid + } +} + +/// Map a `spawn_workload` wait result to a placement-neutral status. A wait +/// error fails the boundary closed (`Exited(1)`) — the error is logged, never +/// cached (`BackendError` is `!Clone`). +fn wait_result_to_status(result: miette::Result) -> BoundaryExitStatus { + match result { + Ok(code) => BoundaryExitStatus::Exited(code), + Err(e) => { + warn!(error = %e, "Sidecar workload wait failed; failing closed"); + BoundaryExitStatus::Exited(1) + } + } +} + +// ============================================================================ +// Exec and mediation ingress +// ============================================================================ + +/// Sidecar exec interface. As with the in-pod backend, the live SSH server still +/// spawns workload shells directly; routing those through an owned +/// [`ExecSession`] is the remaining live-adoption refactor, so this returns an +/// explicit error rather than a half-wired session. +struct SidecarExec; + +#[async_trait] +impl BoundaryExec for SidecarExec { + async fn exec(&self, _spec: ExecSpec) -> Result { + Err(BackendError::Process( + "sidecar BoundaryExec is not yet the live SSH exec path; SSH execs directly." + .to_string(), + )) + } +} + +/// Sidecar mediation ingress. Egress mediation is hosted remotely in the network +/// container, so this boundary hosts none: `accept` fails closed rather than +/// competing with the remote proxy's accept loop. +struct SidecarMediationIngress; + +#[async_trait] +impl MediationIngress for SidecarMediationIngress { + async fn accept(&self) -> Result { + Err(BackendError::Process( + "sidecar MediationIngress hosts no connections; mediation runs in the network container" + .to_string(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use openshell_core::policy::{ + FilesystemPolicy, LandlockPolicy, NetworkMode, NetworkPolicy, ProcessPolicy, SandboxPolicy, + }; + use openshell_isolation::AgentSpec; + use openshell_isolation::contract::{BackendRegistry, TopologyDescriptor}; + + fn minimal_config() -> SidecarConfig { + SidecarConfig { + process_enforcement_mode: ProcessEnforcementMode::NetworkOnly, + provider_credentials: ProviderCredentialState::from_environment( + 0, + HashMap::new(), + HashMap::new(), + HashMap::new(), + ), + provider_env: HashMap::new(), + process_policy: block_mode_policy(), + ca_file_paths: None, + entrypoint_pid: Arc::new(AtomicU32::new(0)), + ssh_socket_path: None, + openshell_endpoint: None, + sandbox_id: None, + process_control_writer: None, + process_control_closed: Mutex::new(None), + #[cfg(target_os = "linux")] + bypass_denial_tx: Mutex::new(None), + #[cfg(target_os = "linux")] + bypass_activity_tx: Mutex::new(None), + } + } + + fn block_mode_policy() -> SandboxPolicy { + SandboxPolicy { + version: 1, + filesystem: FilesystemPolicy::default(), + network: NetworkPolicy { + mode: NetworkMode::Block, + proxy: None, + }, + landlock: LandlockPolicy::default(), + process: ProcessPolicy::default(), + } + } + + fn descriptor() -> TopologyDescriptor { + TopologyDescriptor { + contract_version: CONTRACT_VERSION, + backend_id: SIDECAR_BACKEND_ID.to_string(), + payload: Vec::new(), + } + } + + fn sandbox_context() -> SandboxContext { + SandboxContext { + sandbox_id: "test-sandbox".to_string(), + policy: block_mode_policy(), + agent: AgentSpec { + program: "true".to_string(), + args: vec![], + workdir: None, + timeout_secs: 0, + interactive: false, + }, + } + } + + #[test] + fn factory_speaks_the_contract_version() { + let factory = SidecarBackendFactory::new(minimal_config()); + assert_eq!(factory.backend_id(), SIDECAR_BACKEND_ID); + assert_eq!(factory.contract_version(), CONTRACT_VERSION); + } + + #[test] + fn registry_selects_sidecar_backend() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(SidecarBackendFactory::new(minimal_config()))) + .expect("register"); + let (factory, _verified) = registry + .resolve(descriptor(), SIDECAR_BACKEND_ID) + .expect("resolve"); + assert_eq!(factory.backend_id(), SIDECAR_BACKEND_ID); + } + + /// Drive the sidecar chain attach -> Bound -> confirm -> Ready and prove the + /// retained mediation ingress survives the consuming transitions. + #[tokio::test] + async fn lifecycle_reaches_ready_and_retains_ingress() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(SidecarBackendFactory::new(minimal_config()))) + .expect("register"); + let (factory, verified) = registry + .resolve(descriptor(), SIDECAR_BACKEND_ID) + .expect("resolve"); + + let bound = factory + .attach(verified, sandbox_context()) + .await + .expect("attach"); + let ingress = bound.mediation_ingress(); + let _ready = bound.confirm().await.expect("confirm"); + // Mediation is remote; the local ingress reports an explicit error. + assert!(ingress.accept().await.is_err()); + } + + /// A descriptor that names the sidecar backend but carries an unexpected + /// non-empty payload is a malformed/tampered value and must fail closed at + /// the trust boundary (the payload is validated by the backend in `attach`). + #[tokio::test] + async fn attach_rejects_nonempty_payload() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(SidecarBackendFactory::new(minimal_config()))) + .expect("register"); + let tampered = TopologyDescriptor { + contract_version: CONTRACT_VERSION, + backend_id: SIDECAR_BACKEND_ID.to_string(), + payload: vec![1, 2, 3], + }; + let (factory, verified) = registry + .resolve(tampered, SIDECAR_BACKEND_ID) + .expect("resolve"); + let err = factory + .attach(verified, sandbox_context()) + .await + .map(|_| ()) + .expect_err("non-empty payload must be rejected"); + assert!(matches!(err, BackendError::Descriptor(_))); + } + + /// `confirm` fails closed when the authoritative control link to the network + /// container is already down: the workload must not be launched only to be + /// torn down on the first `wait`. + #[tokio::test] + async fn confirm_fails_closed_when_control_channel_dead() { + let mut config = minimal_config(); + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + // Sender dropped without sending: the control link is gone. + drop(tx); + config.process_control_closed = Mutex::new(Some(rx)); + + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(SidecarBackendFactory::new(config))) + .expect("register"); + let (factory, verified) = registry + .resolve(descriptor(), SIDECAR_BACKEND_ID) + .expect("resolve"); + let bound = factory + .attach(verified, sandbox_context()) + .await + .expect("attach"); + let err = bound + .confirm() + .await + .map(|_| ()) + .expect_err("confirm must fail closed on a dead control channel"); + assert!(matches!(err, BackendError::Confirm(_))); + } + + /// `attach` is atomic and never binds a resource that is already bound to an + /// active boundary: the sidecar resource binds exactly once. + #[tokio::test] + async fn second_attach_is_denied() { + let mut registry = BackendRegistry::new(); + registry + .register(Arc::new(SidecarBackendFactory::new(minimal_config()))) + .expect("register"); + + let (factory, verified) = registry + .resolve(descriptor(), SIDECAR_BACKEND_ID) + .expect("resolve"); + let _bound = factory + .attach(verified, sandbox_context()) + .await + .expect("first attach"); + + let (_factory2, verified2) = registry + .resolve(descriptor(), SIDECAR_BACKEND_ID) + .expect("re-resolve"); + let err = factory + .attach(verified2, sandbox_context()) + .await + .map(|_| ()) + .expect_err("second attach must fail"); + assert!(matches!(err, BackendError::Denied(_))); + } +}