diff --git a/Cargo.lock b/Cargo.lock index f93440a39..572bd5f3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3278,11 +3278,13 @@ dependencies = [ "bollard", "bytes", "futures", + "hmac", "miette", "openshell-core", "rcgen", "serde", "serde_json", + "sha2 0.10.9", "tar", "tempfile", "tokio", diff --git a/architecture/sandbox-custom-containers.md b/architecture/sandbox-custom-containers.md index 5d482ffe0..69e5c10d4 100644 --- a/architecture/sandbox-custom-containers.md +++ b/architecture/sandbox-custom-containers.md @@ -33,8 +33,9 @@ The community registry prefix defaults to `ghcr.io/nvidia/openshell-community/sa When `--from` points to a Dockerfile or directory, the CLI: 1. Builds the image locally via the Docker daemon (respecting `.dockerignore`). -2. Pushes it into the cluster's containerd runtime using `docker save` / `ctr import`. -3. Creates the sandbox with the resulting image tag. +2. For a local Kubernetes gateway, pushes it into the cluster's containerd runtime using `docker save` / `ctr import`. +3. For a local VM gateway, exports the built image as a rootfs tar artifact and passes the VM driver a gateway-signed internal artifact reference. The driver rejects unsigned `openshell-rootfs-tar:` references so API clients cannot point it at arbitrary gateway host files. +4. Creates the sandbox with the resulting image tag or VM artifact reference. ## How It Works diff --git a/crates/openshell-bootstrap/Cargo.toml b/crates/openshell-bootstrap/Cargo.toml index 30fd4fbfc..821817fd8 100644 --- a/crates/openshell-bootstrap/Cargo.toml +++ b/crates/openshell-bootstrap/Cargo.toml @@ -16,10 +16,12 @@ base64 = "0.22" bollard = { version = "0.20", features = ["ssh"] } bytes = { workspace = true } futures = { workspace = true } +hmac = "0.12" miette = { workspace = true } rcgen = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } +sha2 = "0.10" url = { workspace = true } tar = "0.4" tempfile = "3" diff --git a/crates/openshell-bootstrap/src/build.rs b/crates/openshell-bootstrap/src/build.rs index d951f2833..3191b88b1 100644 --- a/crates/openshell-bootstrap/src/build.rs +++ b/crates/openshell-bootstrap/src/build.rs @@ -9,15 +9,21 @@ //! runtime, while the VM backend can export the built image as a rootfs tar. use std::collections::HashMap; +use std::fs::File; +use std::io::Read; use std::path::{Path, PathBuf}; +use base64::Engine as _; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; use bollard::Docker; use bollard::models::ContainerCreateBody; use bollard::query_parameters::{ BuildImageOptionsBuilder, CreateContainerOptionsBuilder, RemoveContainerOptionsBuilder, }; use futures::StreamExt; +use hmac::{Hmac, Mac}; use miette::{IntoDiagnostic, Result, WrapErr}; +use sha2::{Digest, Sha256}; use tokio::io::AsyncWriteExt; use url::{Position, Url}; @@ -26,6 +32,15 @@ use crate::push::push_local_images; /// Pseudo-image URI scheme used to hand a local rootfs tar artifact to the VM driver. pub const ROOTFS_TAR_IMAGE_REF_SCHEME: &str = "openshell-rootfs-tar"; +const ROOTFS_TAR_IMAGE_REF_VERSION: &str = "v1"; +type HmacSha256 = Hmac; + +/// Authenticated VM rootfs tar artifact decoded from an internal image reference. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RootfsTarImageRef { + pub path: PathBuf, + pub digest: String, +} /// Build a container image from a Dockerfile using the local Docker daemon. /// @@ -65,6 +80,25 @@ pub fn encode_rootfs_tar_image_ref(path: &Path) -> Result { )) } +/// Encode a local rootfs tar path as an authenticated VM-driver artifact reference. +pub fn encode_authenticated_rootfs_tar_image_ref(path: &Path, secret: &str) -> Result { + let canonical = path + .canonicalize() + .into_diagnostic() + .wrap_err_with(|| format!("failed to resolve rootfs tar path {}", path.display()))?; + let digest = compute_file_sha256_hex(&canonical)?; + let file_url = Url::from_file_path(&canonical).map_err(|()| { + miette::miette!("failed to encode rootfs tar path {}", canonical.display()) + })?; + let file_url = file_url.to_string(); + let signature = sign_rootfs_tar_image_ref(secret, &file_url, &digest)?; + Ok(format!( + "{ROOTFS_TAR_IMAGE_REF_SCHEME}:{ROOTFS_TAR_IMAGE_REF_VERSION}:{}:{digest}:{}", + URL_SAFE_NO_PAD.encode(file_url.as_bytes()), + URL_SAFE_NO_PAD.encode(signature), + )) +} + /// Decode a VM-driver rootfs tar image reference back to a local filesystem path. pub fn decode_rootfs_tar_image_ref(image_ref: &str) -> Option { let remainder = image_ref.strip_prefix(&format!("{ROOTFS_TAR_IMAGE_REF_SCHEME}:"))?; @@ -72,6 +106,62 @@ pub fn decode_rootfs_tar_image_ref(image_ref: &str) -> Option { Url::parse(&file_url).ok()?.to_file_path().ok() } +/// Decode and verify an authenticated VM-driver rootfs tar image reference. +pub fn decode_authenticated_rootfs_tar_image_ref( + image_ref: &str, + secret: &str, +) -> Result> { + let Some(remainder) = image_ref.strip_prefix(&format!( + "{ROOTFS_TAR_IMAGE_REF_SCHEME}:{ROOTFS_TAR_IMAGE_REF_VERSION}:" + )) else { + return Ok(None); + }; + + let mut parts = remainder.split(':'); + let encoded_url = parts + .next() + .filter(|value| !value.is_empty()) + .ok_or_else(|| miette::miette!("missing rootfs artifact path"))?; + let digest = parts + .next() + .filter(|value| !value.is_empty()) + .ok_or_else(|| miette::miette!("missing rootfs artifact digest"))?; + let encoded_signature = parts + .next() + .filter(|value| !value.is_empty()) + .ok_or_else(|| miette::miette!("missing rootfs artifact signature"))?; + if parts.next().is_some() { + return Err(miette::miette!("malformed rootfs artifact reference")); + } + validate_sha256_hex(digest)?; + + let file_url_bytes = URL_SAFE_NO_PAD + .decode(encoded_url) + .map_err(|err| miette::miette!("invalid rootfs artifact path encoding: {err}"))?; + let file_url = String::from_utf8(file_url_bytes) + .map_err(|err| miette::miette!("invalid rootfs artifact path encoding: {err}"))?; + + let signature = URL_SAFE_NO_PAD + .decode(encoded_signature) + .map_err(|err| miette::miette!("invalid rootfs artifact signature encoding: {err}"))?; + verify_rootfs_tar_image_ref_signature(secret, &file_url, digest, &signature)?; + + let url = Url::parse(&file_url) + .into_diagnostic() + .wrap_err("invalid rootfs artifact URL")?; + if url.scheme() != "file" { + return Err(miette::miette!("rootfs artifact URL must use file scheme")); + } + let path = url + .to_file_path() + .map_err(|()| miette::miette!("rootfs artifact URL is not a local file path"))?; + + Ok(Some(RootfsTarImageRef { + path, + digest: format!("sha256:{digest}"), + })) +} + /// Export a locally-built Docker image as a persistent rootfs tar artifact for the VM driver. pub async fn export_local_image_rootfs( image_ref: &str, @@ -103,6 +193,63 @@ pub async fn export_local_image_rootfs( Ok(output_path) } +fn compute_file_sha256_hex(path: &Path) -> Result { + let mut file = File::open(path) + .into_diagnostic() + .wrap_err_with(|| format!("failed to open {}", path.display()))?; + let mut hasher = Sha256::new(); + let mut buf = [0u8; 64 * 1024]; + loop { + let read = file + .read(&mut buf) + .into_diagnostic() + .wrap_err_with(|| format!("failed to read {}", path.display()))?; + if read == 0 { + break; + } + hasher.update(&buf[..read]); + } + Ok(format!("{:x}", hasher.finalize())) +} + +fn validate_sha256_hex(value: &str) -> Result<()> { + if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) { + Ok(()) + } else { + Err(miette::miette!("rootfs artifact digest must be sha256 hex")) + } +} + +fn rootfs_tar_image_ref_message(file_url: &str, digest: &str) -> String { + format!("{ROOTFS_TAR_IMAGE_REF_VERSION}\n{file_url}\nsha256:{digest}") +} + +fn rootfs_tar_image_ref_hmac(secret: &str) -> Result { + if secret.trim().is_empty() { + return Err(miette::miette!("rootfs artifact secret is empty")); + } + HmacSha256::new_from_slice(secret.as_bytes()) + .map_err(|err| miette::miette!("invalid rootfs artifact secret: {err}")) +} + +fn sign_rootfs_tar_image_ref(secret: &str, file_url: &str, digest: &str) -> Result> { + let mut mac = rootfs_tar_image_ref_hmac(secret)?; + mac.update(rootfs_tar_image_ref_message(file_url, digest).as_bytes()); + Ok(mac.finalize().into_bytes().to_vec()) +} + +fn verify_rootfs_tar_image_ref_signature( + secret: &str, + file_url: &str, + digest: &str, + signature: &[u8], +) -> Result<()> { + let mut mac = rootfs_tar_image_ref_hmac(secret)?; + mac.update(rootfs_tar_image_ref_message(file_url, digest).as_bytes()); + mac.verify_slice(signature) + .map_err(|_| miette::miette!("rootfs artifact signature verification failed")) +} + /// Push a locally-built image into the gateway's containerd runtime. #[allow(clippy::implicit_hasher)] pub async fn push_image_into_gateway( @@ -639,4 +786,35 @@ mod tests { assert_eq!(decoded, tar_path.canonicalize().unwrap()); } + + #[test] + fn authenticated_rootfs_tar_image_ref_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let tar_path = dir.path().join("rootfs tar.tar"); + fs::write(&tar_path, "rootfs").unwrap(); + + let encoded = encode_authenticated_rootfs_tar_image_ref(&tar_path, "secret").unwrap(); + let decoded = decode_authenticated_rootfs_tar_image_ref(&encoded, "secret") + .unwrap() + .unwrap(); + + assert_eq!(decoded.path, tar_path.canonicalize().unwrap()); + assert_eq!( + decoded.digest, + "sha256:3c47ef972d531d524daa15fa33dd885dd23de6221bbd10a29eb42ecfcf2ef422" + ); + } + + #[test] + fn authenticated_rootfs_tar_image_ref_rejects_wrong_secret() { + let dir = tempfile::tempdir().unwrap(); + let tar_path = dir.path().join("rootfs.tar"); + fs::write(&tar_path, "rootfs").unwrap(); + + let encoded = encode_authenticated_rootfs_tar_image_ref(&tar_path, "secret").unwrap(); + let err = decode_authenticated_rootfs_tar_image_ref(&encoded, "wrong-secret") + .expect_err("wrong secret should fail"); + + assert!(err.to_string().contains("signature verification failed")); + } } diff --git a/crates/openshell-bootstrap/src/metadata.rs b/crates/openshell-bootstrap/src/metadata.rs index 41e75e811..5892dd15f 100644 --- a/crates/openshell-bootstrap/src/metadata.rs +++ b/crates/openshell-bootstrap/src/metadata.rs @@ -46,6 +46,14 @@ pub struct GatewayMetadata { alias = "cf_auth_url" )] pub edge_auth_url: Option, + + /// Local VM driver state directory for standalone VM gateways. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_driver_state_dir: Option, + + /// Local secret used to authenticate VM rootfs artifact references. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vm_rootfs_artifact_secret: Option, } impl GatewayMetadata { @@ -136,6 +144,8 @@ pub fn create_gateway_metadata_with_host( auth_mode: disable_tls.then(|| "plaintext".to_string()), edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, } } @@ -463,6 +473,8 @@ mod tests { auth_mode: None, edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }; let json = serde_json::to_string(&meta).unwrap(); let parsed: GatewayMetadata = serde_json::from_str(&json).unwrap(); @@ -558,6 +570,8 @@ mod tests { auth_mode: None, edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }; assert_eq!(meta.gateway_host(), None); } @@ -574,6 +588,8 @@ mod tests { auth_mode: None, edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }; assert_eq!(meta.gateway_host(), Some("10.0.0.5")); } diff --git a/crates/openshell-cli/src/completers.rs b/crates/openshell-cli/src/completers.rs index c8b5c82a3..31a98158c 100644 --- a/crates/openshell-cli/src/completers.rs +++ b/crates/openshell-cli/src/completers.rs @@ -183,6 +183,8 @@ mod tests { auth_mode: Some("cloudflare_jwt".to_string()), edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }, ) .unwrap(); diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index 57f3dbc84..da7b3d13a 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -2845,6 +2845,8 @@ mod tests { auth_mode: Some("cloudflare_jwt".to_string()), edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, } } diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index f47de664b..edb3da982 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -194,46 +194,6 @@ enum KubeEventReason { Started, } -/// Decide whether a server/driver log line should be surfaced during -/// `sandbox create` provisioning. -/// -/// Defaults to a curated allowlist so the spinner stays focused on -/// meaningful progress. Set `OPENSHELL_PROVISION_VERBOSE=1` to surface -/// every log line (helpful when debugging stuck provisioning, e.g. on -/// the experimental VM gateway). -fn should_show_provisioning_log(line: &openshell_core::proto::SandboxLogLine) -> bool { - // Allowlisted substrings for info-level progress lines emitted by - // the server compute layer and bundled drivers (VM / Docker). Keep - // this short — anything not matching is suppressed by default. - const ALLOWLIST: &[&str] = &[ - "Sandbox phase changed", - "Pulling image", - "Pulled image", - "Extracting", - "Preparing rootfs", - "Booting VM", - "Starting VM", - "Starting sandbox", - "Sandbox ready", - "Supervisor connected", - ]; - - if std::env::var("OPENSHELL_PROVISION_VERBOSE") - .map(|v| !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false")) - .unwrap_or(false) - { - return true; - } - - // Always surface warnings and errors during provisioning. - let level = line.level.to_ascii_lowercase(); - if matches!(level.as_str(), "warn" | "warning" | "error") { - return true; - } - - ALLOWLIST.iter().any(|needle| line.message.contains(needle)) -} - /// Map a Kubernetes event reason string to an enum. fn parse_kube_event_reason(reason: &str) -> Option { match reason { @@ -937,6 +897,8 @@ fn plaintext_gateway_metadata( auth_mode: Some("plaintext".to_string()), edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, } } @@ -1138,6 +1100,8 @@ pub async fn gateway_add( auth_mode: Some("mtls".to_string()), edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }; store_gateway_metadata(name, &metadata)?; @@ -1167,6 +1131,8 @@ pub async fn gateway_add( auth_mode: Some("cloudflare_jwt".to_string()), edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }; store_gateway_metadata(name, &metadata)?; @@ -2342,31 +2308,6 @@ pub async fn sandbox_create( if !saw_gateway_ready && line.message.contains("listening") { saw_gateway_ready = true; } - - // Surface log lines as progress so users aren't staring at a - // silent spinner while non-Kubernetes drivers (VM, Docker) do - // their work. Drivers/server tracing with a `sandbox_id` - // field flows through here as Log payloads. - // - // The default filter keeps output focused on user-relevant - // progress: warn/error always, plus a curated allowlist of - // info messages. Set OPENSHELL_PROVISION_VERBOSE=1 to see - // every log line during provisioning. - if !line.message.is_empty() && should_show_provisioning_log(&line) { - if let Some(d) = display.as_mut() { - // Interactive: tuck the message under the spinner - // as detail so the checklist stays clean. - d.set_active_detail(&line.message); - } else { - let ts = format_timestamp(provision_start.elapsed()); - let level = if line.level.is_empty() { - "INFO".to_string() - } else { - line.level.to_uppercase() - }; - eprintln!(" {} {} {}", ts.dimmed(), level.dimmed(), line.message,); - } - } } Some(openshell_core::proto::sandbox_stream_event::Payload::Event(ev)) => { // Map Kubernetes events to provisioning steps. @@ -2794,7 +2735,20 @@ async fn build_from_dockerfile( let rootfs_tar = openshell_bootstrap::build::export_local_image_rootfs(&tag, &mut on_log) .await .wrap_err("failed to export built image as a VM rootfs artifact")?; - let artifact_ref = openshell_bootstrap::build::encode_rootfs_tar_image_ref(&rootfs_tar)?; + let artifact_secret = metadata + .as_ref() + .and_then(|metadata| metadata.vm_rootfs_artifact_secret.as_deref()) + .filter(|secret| !secret.trim().is_empty()) + .ok_or_else(|| { + miette!( + "local Dockerfile sources for VM gateways require authenticated rootfs artifact metadata; restart gateway '{}' with a current `mise run gateway:vm`", + gateway_name + ) + })?; + let artifact_ref = openshell_bootstrap::build::encode_authenticated_rootfs_tar_image_ref( + &rootfs_tar, + artifact_secret, + )?; eprintln!(); eprintln!( @@ -5705,6 +5659,8 @@ mod tests { auth_mode: Some("cloudflare_jwt".to_string()), edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, } } @@ -5890,6 +5846,8 @@ mod tests { auth_mode: None, edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }; assert!(!dockerfile_sources_supported_for_gateway(Some(&metadata))); @@ -5907,6 +5865,8 @@ mod tests { auth_mode: None, edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }; assert!(dockerfile_sources_supported_for_gateway(Some(&metadata))); @@ -6128,6 +6088,8 @@ mod tests { auth_mode: None, edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }, ]; @@ -6163,6 +6125,8 @@ mod tests { auth_mode: None, edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, + vm_rootfs_artifact_secret: None, }; assert_eq!(gateway_auth_label(&gateway), "mtls"); diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 6fc27dd69..0270c5a5d 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -17,7 +17,9 @@ use oci_client::client::{Client as OciClient, ClientConfig}; use oci_client::manifest::{ImageIndexEntry, OciDescriptor}; use oci_client::secrets::RegistryAuth; use oci_client::{Reference, RegistryOperation}; -use openshell_bootstrap::build::decode_rootfs_tar_image_ref; +use openshell_bootstrap::build::{ + ROOTFS_TAR_IMAGE_REF_SCHEME, decode_authenticated_rootfs_tar_image_ref, +}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, @@ -116,6 +118,7 @@ pub struct VmDriverConfig { pub gpu_enabled: bool, pub gpu_mem_mib: u32, pub gpu_vcpus: u8, + pub rootfs_artifact_secret: Option, } impl Default for VmDriverConfig { @@ -137,6 +140,7 @@ impl Default for VmDriverConfig { gpu_enabled: false, gpu_mem_mib: 8192, gpu_vcpus: 4, + rootfs_artifact_secret: None, } } } @@ -756,9 +760,32 @@ impl VmDriver { sandbox_id: &str, image_ref: &str, ) -> Result { - if let Some(rootfs_tar_path) = decode_rootfs_tar_image_ref(image_ref) { + if is_rootfs_tar_image_ref(image_ref) { + let secret = self + .config + .rootfs_artifact_secret + .as_deref() + .filter(|secret| !secret.trim().is_empty()) + .ok_or_else(|| { + Status::failed_precondition( + "vm rootfs tar image references require a gateway-issued artifact secret", + ) + })?; + let artifact = decode_authenticated_rootfs_tar_image_ref(image_ref, secret) + .map_err(|err| { + Status::failed_precondition(format!( + "invalid vm rootfs artifact reference: {err}" + )) + })? + .ok_or_else(|| { + Status::failed_precondition("invalid vm rootfs artifact reference") + })?; return self - .ensure_cached_rootfs_tar_image_rootfs_archive(image_ref, &rootfs_tar_path) + .ensure_cached_rootfs_tar_image_rootfs_archive( + image_ref, + &artifact.path, + &artifact.digest, + ) .await; } @@ -853,6 +880,7 @@ impl VmDriver { &self, image_ref: &str, rootfs_tar_path: &Path, + expected_digest: &str, ) -> Result { let rootfs_tar = rootfs_tar_path.to_path_buf(); let image_identity = tokio::task::spawn_blocking(move || compute_file_sha256(&rootfs_tar)) @@ -866,6 +894,11 @@ impl VmDriver { rootfs_tar_path.display() )) })?; + if image_identity != expected_digest { + return Err(Status::failed_precondition( + "vm rootfs artifact digest does not match the authenticated reference", + )); + } let archive_path = image_cache_rootfs_archive(&self.config.state_dir, &image_identity); if tokio::fs::metadata(&archive_path).await.is_ok() { @@ -1403,6 +1436,12 @@ fn parse_registry_reference(image_ref: &str) -> Result { }) } +fn is_rootfs_tar_image_ref(image_ref: &str) -> bool { + image_ref + .strip_prefix(ROOTFS_TAR_IMAGE_REF_SCHEME) + .is_some_and(|rest| rest.starts_with(':')) +} + fn registry_client() -> OciClient { OciClient::new(ClientConfig { platform_resolver: Some(Box::new(linux_platform_resolver)), @@ -1470,8 +1509,12 @@ fn env_non_empty(key: &str) -> Option { } fn image_reference_registry_host(image_ref: &str) -> &str { - let first = image_ref.split('/').next().unwrap_or(image_ref); - if first.contains('.') || first.contains(':') || first.eq_ignore_ascii_case("localhost") { + let mut parts = image_ref.splitn(2, '/'); + let first = parts.next().unwrap_or(image_ref); + let has_path = parts.next().is_some(); + if has_path + && (first.contains('.') || first.contains(':') || first.eq_ignore_ascii_case("localhost")) + { first } else { "docker.io" @@ -2524,16 +2567,49 @@ mod tests { #[test] fn image_reference_registry_host_defaults_to_docker_hub() { assert_eq!(image_reference_registry_host("ubuntu:24.04"), "docker.io"); + assert_eq!( + image_reference_registry_host("library/ubuntu:24.04"), + "docker.io" + ); assert_eq!( image_reference_registry_host("ghcr.io/nvidia/openshell/base:latest"), "ghcr.io" ); + assert_eq!( + image_reference_registry_host("localhost/example:dev"), + "localhost" + ); assert_eq!( image_reference_registry_host("localhost:5000/example/sandbox:dev"), "localhost:5000" ); } + #[tokio::test] + async fn ensure_cached_image_rootfs_archive_rejects_unsigned_rootfs_tar_refs() { + let base = unique_temp_dir(); + fs::create_dir_all(&base).unwrap(); + let artifact = base.join("rootfs.tar"); + fs::write(&artifact, "not a real rootfs").unwrap(); + let image_ref = openshell_bootstrap::build::encode_rootfs_tar_image_ref(&artifact).unwrap(); + let driver = test_driver(VmDriverConfig { + state_dir: base.join("driver-state"), + ..Default::default() + }); + + let err = driver + .ensure_cached_image_rootfs_archive("sandbox-123", &image_ref) + .await + .expect_err("unsigned rootfs tar refs must be rejected"); + + assert_eq!(err.code(), Code::FailedPrecondition); + assert!( + err.message().contains("gateway-issued artifact secret"), + "unexpected error: {}", + err.message() + ); + } + #[test] fn apply_layer_dir_to_rootfs_honors_whiteouts() { let base = unique_temp_dir(); @@ -2823,6 +2899,21 @@ mod tests { )) } + fn test_driver(config: VmDriverConfig) -> VmDriver { + VmDriver { + config, + launcher_bin: PathBuf::from("openshell-driver-vm"), + registry: Arc::new(Mutex::new(HashMap::new())), + image_cache_lock: Arc::new(Mutex::new(())), + events: broadcast::channel(WATCH_BUFFER).0, + gpu_inventory: None, + subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new( + Ipv4Addr::new(10, 0, 128, 0), + 17, + ))), + } + } + fn spawn_exited_child() -> Child { Command::new("sh") .arg("-c") diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 596e6c88d..3f5d70c14 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -105,6 +105,9 @@ struct Args { #[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)] gpu_vcpus: u8, + #[arg(long, env = "OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET", hide = true)] + rootfs_artifact_secret: Option, + #[arg(long, hide = true)] vm_backend: Option, @@ -185,6 +188,7 @@ async fn main() -> Result<()> { gpu_enabled: args.gpu, gpu_mem_mib: args.gpu_mem_mib, gpu_vcpus: args.gpu_vcpus, + rootfs_artifact_secret: args.rootfs_artifact_secret, }) .await .map_err(|err| miette::miette!("{err}"))?; diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 6c59e9822..456d885fe 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -148,10 +148,14 @@ pub async fn run_server( if database_url.is_empty() { return Err(Error::config("database_url is required")); } - // The ssh_handshake_secret check is driver-scoped: the Kubernetes and - // Podman drivers inject the secret into sandbox env, while the VM and - // Docker drivers do not consume it. The check is enforced per-driver - // in `build_compute_runtime`. + let driver = configured_compute_driver(&config)?; + if config.ssh_handshake_secret.is_empty() + && !matches!(driver, ComputeDriverKind::Docker | ComputeDriverKind::Vm) + { + return Err(Error::config( + "ssh_handshake_secret is required. Set --ssh-handshake-secret or OPENSHELL_SSH_HANDSHAKE_SECRET", + )); + } let store = Arc::new(Store::connect(database_url).await?); @@ -366,35 +370,32 @@ async fn build_compute_runtime( info!(driver = %driver, "Using compute driver"); match driver { - ComputeDriverKind::Kubernetes => { - require_ssh_handshake_secret(config)?; - ComputeRuntime::new_kubernetes( - KubernetesComputeConfig { - namespace: config.sandbox_namespace.clone(), - default_image: config.sandbox_image.clone(), - image_pull_policy: config.sandbox_image_pull_policy.clone(), - grpc_endpoint: config.grpc_endpoint.clone(), - // Filesystem path to the supervisor's Unix-socket SSH daemon. - // The path lives in a root-only directory so only the - // supervisor can connect; the gateway reaches it through - // the RelayStream bridge, not directly. Override via - // `sandbox_ssh_socket_path` in the config for deployments - // where multiple supervisors share a filesystem. - ssh_socket_path: config.sandbox_ssh_socket_path.clone(), - ssh_handshake_secret: config.ssh_handshake_secret.clone(), - ssh_handshake_skew_secs: config.ssh_handshake_skew_secs, - client_tls_secret_name: config.client_tls_secret_name.clone(), - host_gateway_ip: config.host_gateway_ip.clone(), - }, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions.clone(), - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) - } + ComputeDriverKind::Kubernetes => ComputeRuntime::new_kubernetes( + KubernetesComputeConfig { + namespace: config.sandbox_namespace.clone(), + default_image: config.sandbox_image.clone(), + image_pull_policy: config.sandbox_image_pull_policy.clone(), + grpc_endpoint: config.grpc_endpoint.clone(), + // Filesystem path to the supervisor's Unix-socket SSH daemon. + // The path lives in a root-only directory so only the + // supervisor can connect; the gateway reaches it through the + // RelayStream bridge, not directly. Override via + // `sandbox_ssh_socket_path` in the config for deployments + // where multiple supervisors share a filesystem. + ssh_socket_path: config.sandbox_ssh_socket_path.clone(), + ssh_handshake_secret: config.ssh_handshake_secret.clone(), + ssh_handshake_skew_secs: config.ssh_handshake_skew_secs, + client_tls_secret_name: config.client_tls_secret_name.clone(), + host_gateway_ip: config.host_gateway_ip.clone(), + }, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions.clone(), + ) + .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))), ComputeDriverKind::Docker => ComputeRuntime::new_docker( config.clone(), docker_config.clone(), @@ -421,7 +422,6 @@ async fn build_compute_runtime( .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) } ComputeDriverKind::Podman => { - require_ssh_handshake_secret(config)?; let socket_path = std::env::var("OPENSHELL_PODMAN_SOCKET") .ok() .filter(|s| !s.is_empty()) @@ -473,15 +473,6 @@ async fn build_compute_runtime( } } -fn require_ssh_handshake_secret(config: &Config) -> Result<()> { - if config.ssh_handshake_secret.is_empty() { - return Err(Error::config( - "ssh_handshake_secret is required for this driver. Set --ssh-handshake-secret or OPENSHELL_SSH_HANDSHAKE_SECRET", - )); - } - Ok(()) -} - fn configured_compute_driver(config: &Config) -> Result { match config.compute_drivers.as_slice() { [] => Err(Error::config( diff --git a/tasks/gateway.toml b/tasks/gateway.toml index 23df38b3a..1df07cb95 100644 --- a/tasks/gateway.toml +++ b/tasks/gateway.toml @@ -2,26 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # Standalone gateway tasks -# -# Canonical entry point: -# mise run gateway:start # driver = docker | vm -# -# Backward-compatible aliases: -# mise run gateway:docker -# mise run gateway:vm -# -# All three commands also write ~/.config/openshell/active_gateway so the -# `openshell` CLI automatically targets the gateway you just started — no -# need to follow up with `openshell gateway select`. Inside this repo you -# can override the active gateway per-developer by setting OPENSHELL_GATEWAY -# in `.env` (mise loads it automatically). - -["gateway:start"] -description = "Run a standalone gateway with the given compute driver (docker | vm)" -usage = """ -arg "[driver]" help="Compute driver to start (docker | vm). Defaults to $OPENSHELL_GATEWAY_DRIVER or 'docker'." -""" -run = "bash tasks/scripts/gateway-start.sh" ["gateway:docker"] description = "Run a standalone gateway with the bundled Docker compute driver" diff --git a/tasks/scripts/gateway-docker.sh b/tasks/scripts/gateway-docker.sh index 44e2ec7c7..23527741f 100644 --- a/tasks/scripts/gateway-docker.sh +++ b/tasks/scripts/gateway-docker.sh @@ -6,10 +6,6 @@ # Start a standalone openshell-gateway backed by the Docker compute driver for # local manual testing. # -# Invocation: -# mise run gateway:start docker # canonical -# mise run gateway:docker # alias -# # Defaults: # - Plaintext HTTP on 127.0.0.1:18080 # - Dedicated sandbox namespace "docker-dev" @@ -21,11 +17,9 @@ # OPENSHELL_SANDBOX_NAMESPACE=my-ns mise run gateway:docker # OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:docker # -# This script also writes ~/.config/openshell/active_gateway so the -# `openshell` CLI automatically targets this gateway in subsequent shells. -# No need to run `openshell gateway select`. Inside this repo you can -# override per-developer with OPENSHELL_GATEWAY in `.env` (mise loads it). -# An explicit `--gateway` / `--gateway-endpoint` flag still wins. +# After the gateway is running, point the CLI at it with either: +# openshell --gateway docker-dev +# openshell gateway use docker-dev # then plain `openshell ` set -euo pipefail @@ -92,18 +86,6 @@ register_gateway_metadata() { EOF } -# Mirror what `openshell gateway select ` does: write the gateway name -# to $XDG_CONFIG_HOME/openshell/active_gateway. The CLI picks it up as the -# default target when neither --gateway nor OPENSHELL_GATEWAY is set. -save_active_gateway() { - local name=$1 - local config_home active_gateway_path - config_home="${XDG_CONFIG_HOME:-${HOME}/.config}" - active_gateway_path="${config_home}/openshell/active_gateway" - mkdir -p "$(dirname "${active_gateway_path}")" - printf '%s' "${name}" >"${active_gateway_path}" -} - if [[ ! "${GATEWAY_NAME}" =~ ^[A-Za-z0-9._-]+$ ]]; then echo "ERROR: OPENSHELL_DOCKER_GATEWAY_NAME must contain only letters, numbers, dots, underscores, or dashes" >&2 exit 2 @@ -179,7 +161,6 @@ mkdir -p "${STATE_DIR}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" -save_active_gateway "${GATEWAY_NAME}" echo "Starting standalone Docker gateway..." echo " gateway: ${GATEWAY_NAME}" @@ -187,9 +168,9 @@ echo " endpoint: ${GATEWAY_ENDPOINT}" echo " namespace: ${SANDBOX_NAMESPACE}" echo " state dir: ${STATE_DIR}" echo -echo "Active gateway set to '${GATEWAY_NAME}'. The CLI now targets this gateway" -echo "by default — just run \`openshell \`. Override with --gateway" -echo "or by setting OPENSHELL_GATEWAY (e.g. in .env)." +echo "Point the CLI at this gateway with one of:" +echo " openshell --gateway ${GATEWAY_NAME} status" +echo " openshell gateway select ${GATEWAY_NAME}" echo exec "${GATEWAY_BIN}" \ diff --git a/tasks/scripts/gateway-start.sh b/tasks/scripts/gateway-start.sh deleted file mode 100755 index ee160db8d..000000000 --- a/tasks/scripts/gateway-start.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -# Dispatcher for `mise run gateway:start `. -# -# Usage: -# mise run gateway:start # uses $OPENSHELL_GATEWAY_DRIVER or 'docker' -# mise run gateway:start docker -# mise run gateway:start vm -# -# This is a thin shim that forwards to the driver-specific script. The -# `gateway:docker` / `gateway:vm` mise aliases call those scripts directly. - -set -euo pipefail - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" - -DRIVER="${1:-${OPENSHELL_GATEWAY_DRIVER:-docker}}" - -case "${DRIVER}" in - docker) - exec bash "${ROOT}/tasks/scripts/gateway-docker.sh" - ;; - vm) - exec bash "${ROOT}/tasks/scripts/gateway-vm.sh" - ;; - *) - echo "ERROR: unknown gateway driver '${DRIVER}' (expected 'docker' or 'vm')" >&2 - echo "Usage: mise run gateway:start " >&2 - exit 2 - ;; -esac diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index 008f23b65..e67f61347 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -7,8 +7,7 @@ # (openshell-driver-vm) for local manual testing. # # Invocation: -# mise run gateway:start vm # canonical -# mise run gateway:vm # alias +# mise run gateway:vm # # Defaults: # - Plaintext HTTP on 127.0.0.1:18081 @@ -23,6 +22,7 @@ # OPENSHELL_VM_GATEWAY_NAME=my-vm-gateway mise run gateway:vm # OPENSHELL_SANDBOX_NAMESPACE=my-ns mise run gateway:vm # OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:vm +# mise run gateway:vm -- --gpu # # This script also writes ~/.config/openshell/active_gateway so the # `openshell` CLI automatically targets this gateway in subsequent shells. @@ -84,21 +84,27 @@ register_gateway_metadata() { local name=$1 local endpoint=$2 local port=$3 + local vm_driver_state_dir=$4 + local rootfs_artifact_secret=$5 local config_home gateway_dir config_home="${XDG_CONFIG_HOME:-${HOME}/.config}" gateway_dir="${config_home}/openshell/gateways/${name}" mkdir -p "${gateway_dir}" + chmod 700 "${gateway_dir}" 2>/dev/null || true cat >"${gateway_dir}/metadata.json" </dev/null || true } # Mirror what `openshell gateway select ` does: write the gateway name @@ -146,6 +152,51 @@ check_supervisor_cross_toolchain() { fi } +generate_rootfs_artifact_secret() { + od -An -N32 -tx1 /dev/urandom | tr -d ' \n' +} + +VM_GPU="$(normalize_bool "${OPENSHELL_VM_GPU:-false}")" + +while [ "$#" -gt 0 ]; do + case "$1" in + --gpu) + VM_GPU="true" + shift + ;; + --gpu-mem-mib) + if [ "$#" -lt 2 ]; then + echo "ERROR: --gpu-mem-mib requires a value" >&2 + exit 2 + fi + export OPENSHELL_VM_GPU_MEM_MIB="$2" + shift 2 + ;; + --gpu-vcpus) + if [ "$#" -lt 2 ]; then + echo "ERROR: --gpu-vcpus requires a value" >&2 + exit 2 + fi + export OPENSHELL_VM_GPU_VCPUS="$2" + shift 2 + ;; + -h|--help) + echo "Usage: mise run gateway:vm -- [--gpu] [--gpu-mem-mib MIB] [--gpu-vcpus N]" + exit 0 + ;; + *) + echo "ERROR: unknown gateway-vm option '$1'" >&2 + exit 2 + ;; + esac +done + +if [ "${VM_GPU}" = "true" ]; then + export OPENSHELL_VM_GPU="true" +else + unset OPENSHELL_VM_GPU +fi + if [[ ! "${GATEWAY_NAME}" =~ ^[A-Za-z0-9._-]+$ ]]; then echo "ERROR: OPENSHELL_VM_GATEWAY_NAME must contain only letters, numbers, dots, underscores, or dashes" >&2 exit 2 @@ -168,6 +219,12 @@ VM_DRIVER_STATE_DIR_DEFAULT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}/openshell-v VM_DRIVER_STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${VM_DRIVER_STATE_DIR_DEFAULT}}" DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")" +ROOTFS_ARTIFACT_SECRET="${OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET:-$(generate_rootfs_artifact_secret)}" +if [[ ! "${ROOTFS_ARTIFACT_SECRET}" =~ ^[A-Za-z0-9._~=-]+$ ]]; then + echo "ERROR: OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET must contain only URL-safe characters" >&2 + exit 2 +fi +export OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET="${ROOTFS_ARTIFACT_SECRET}" # Build prerequisites: VM runtime artifacts + bundled supervisor. if [ ! -d "${COMPRESSED_DIR}" ] \ @@ -207,7 +264,7 @@ mkdir -p "${STATE_DIR}" mkdir -p "${VM_DRIVER_STATE_DIR}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" -register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" +register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}" "${ROOTFS_ARTIFACT_SECRET}" save_active_gateway "${GATEWAY_NAME}" echo "Starting standalone VM gateway..." @@ -217,6 +274,7 @@ echo " namespace: ${SANDBOX_NAMESPACE}" echo " state dir: ${STATE_DIR}" echo " driver: ${DRIVER_DIR}/openshell-driver-vm" echo " driver dir: ${VM_DRIVER_STATE_DIR}" +echo " gpu: ${VM_GPU}" echo " image: ${SANDBOX_IMAGE}" echo echo "Active gateway set to '${GATEWAY_NAME}'. The CLI now targets this gateway"