From 3032cc47a191e45fcb3200224c9704ca48cbeaae Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Wed, 29 Apr 2026 20:44:27 -0700 Subject: [PATCH] refactor(vm): move local image handling into driver --- Cargo.lock | 5 +- architecture/custom-vm-runtime.md | 23 +- architecture/sandbox-custom-containers.md | 11 +- crates/openshell-bootstrap/Cargo.toml | 3 - crates/openshell-bootstrap/src/build.rs | 333 +--------------- crates/openshell-bootstrap/src/metadata.rs | 8 - crates/openshell-cli/src/completers.rs | 1 - crates/openshell-cli/src/main.rs | 1 - crates/openshell-cli/src/run.rs | 52 ++- .../sandbox_create_lifecycle_integration.rs | 6 +- crates/openshell-driver-vm/Cargo.toml | 2 +- crates/openshell-driver-vm/README.md | 4 +- crates/openshell-driver-vm/src/driver.rs | 355 ++++++++++++------ crates/openshell-driver-vm/src/main.rs | 4 - crates/openshell-vm/src/lib.rs | 1 + docs/sandboxes/manage-sandboxes.mdx | 4 + tasks/scripts/gateway-vm.sh | 16 +- 17 files changed, 303 insertions(+), 526 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 572bd5f3f..168972961 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3278,18 +3278,15 @@ dependencies = [ "bollard", "bytes", "futures", - "hmac", "miette", "openshell-core", "rcgen", "serde", "serde_json", - "sha2 0.10.9", "tar", "tempfile", "tokio", "tracing", - "url", ] [[package]] @@ -3422,6 +3419,7 @@ dependencies = [ name = "openshell-driver-vm" version = "0.0.0" dependencies = [ + "bollard", "clap", "flate2", "futures", @@ -3430,7 +3428,6 @@ dependencies = [ "miette", "nix", "oci-client", - "openshell-bootstrap", "openshell-core", "openshell-vfio", "polling", diff --git a/architecture/custom-vm-runtime.md b/architecture/custom-vm-runtime.md index 3c721ae50..55f9a8b1f 100644 --- a/architecture/custom-vm-runtime.md +++ b/architecture/custom-vm-runtime.md @@ -76,9 +76,12 @@ Old runtime cache versions are cleaned up when a new version is extracted. ### Sandbox rootfs preparation Each VM sandbox starts from either a registry image fetched directly over OCI or -a local rootfs tar artifact exported by the CLI for Dockerfile-based `--from` -sources, then the driver **rewrites that filesystem into a supervisor-only -sandbox guest** before caching it: +a local Docker image reference produced by Dockerfile-based `--from` sources. +For local Dockerfile sources, the CLI builds the image on the local Docker +daemon and passes the VM driver an internal `openshell-vm-local-image:` +reference. The driver resolves that tag on the gateway host, exports the image +filesystem, and **rewrites that filesystem into a supervisor-only sandbox +guest** before caching it: - `/srv/openshell-vm-sandbox-init.sh` is installed as the guest entrypoint - the bundled `openshell-sandbox` binary is copied into @@ -108,7 +111,7 @@ sandbox image. The driver: - resolves the image on the gateway host without Docker for registry and community image refs - for local Dockerfile sources, the CLI builds through the host Docker socket - and hands the VM driver a local rootfs tar artifact instead of a Docker tag + and hands the VM driver an internal local-image ref instead of a registry ref - unpacks the image filesystem, injects the VM sandbox init/supervisor files, and validates required guest tools such as `bash`, `mount`, `ip`, and `sed` - caches the prepared guest rootfs under @@ -116,16 +119,20 @@ sandbox image. The driver: - extracts a private runtime copy under `/sandboxes//rootfs` -The cache key uses an immutable image identity: repo digest when available, -otherwise a SHA-256 fingerprint of the local rootfs tar artifact. +The cache key uses an immutable image identity: repo digest for registry images +and the local Docker image ID for local-image refs. Different VM sandboxes can use different base images concurrently because the shared cache is per image, not global for the driver. Cached prepared rootfs entries remain on disk until the operator removes them from the VM driver state directory. Docker is therefore no longer required for VM sandboxes created from registry or -community image refs. It is only required on the CLI host when the source is a -local Dockerfile or build context. +community image refs. It is only required on the local CLI/gateway host when the +source is a local Dockerfile or build context. + +Local Dockerfile sources are treated as trusted local-development inputs for VM +gateways. Remote VM gateways still reject local Dockerfile sources until a +gateway-side artifact validation and transfer boundary is designed. There is no embedded guest rootfs fallback anymore. VM sandboxes therefore require either `template.image` or a configured default sandbox image. This is diff --git a/architecture/sandbox-custom-containers.md b/architecture/sandbox-custom-containers.md index 69e5c10d4..cb1546db2 100644 --- a/architecture/sandbox-custom-containers.md +++ b/architecture/sandbox-custom-containers.md @@ -9,7 +9,7 @@ The `--from` flag accepts four kinds of input: | Input | Example | Behavior | |-------|---------|----------| | **Community sandbox name** | `--from openclaw` | Resolves to `ghcr.io/nvidia/openshell-community/sandboxes/openclaw:latest` | -| **Dockerfile path** | `--from ./Dockerfile` | Builds the image, pushes it into the cluster, then creates the sandbox | +| **Dockerfile path** | `--from ./Dockerfile` | Builds the image locally, makes it available to the local gateway, then creates the sandbox | | **Directory with Dockerfile** | `--from ./my-sandbox/` | Uses the directory as the build context | | **Full image reference** | `--from myregistry.com/img:tag` | Uses the image directly | @@ -34,8 +34,13 @@ When `--from` points to a Dockerfile or directory, the CLI: 1. Builds the image locally via the Docker daemon (respecting `.dockerignore`). 2. For a local Kubernetes gateway, pushes it into the cluster's containerd runtime using `docker save` / `ctr import`. -3. For a local VM gateway, exports the built image as a rootfs tar artifact and passes the VM driver a gateway-signed internal artifact reference. The driver rejects unsigned `openshell-rootfs-tar:` references so API clients cannot point it at arbitrary gateway host files. -4. Creates the sandbox with the resulting image tag or VM artifact reference. +3. For a local VM gateway, passes the VM driver an internal `openshell-vm-local-image:` reference. The driver resolves that tag against the local Docker daemon, exports the image filesystem, and prepares the VM rootfs in its own cache. +4. Creates the sandbox with the resulting image tag or VM-local image reference. + +Local Dockerfile sources for VM gateways are trusted local-development inputs. +Remote gateways continue to reject local Dockerfile sources because the gateway +API does not yet validate or transfer local build artifacts across that +boundary. ## How It Works diff --git a/crates/openshell-bootstrap/Cargo.toml b/crates/openshell-bootstrap/Cargo.toml index 821817fd8..942ffc48b 100644 --- a/crates/openshell-bootstrap/Cargo.toml +++ b/crates/openshell-bootstrap/Cargo.toml @@ -16,13 +16,10 @@ base64 = "0.22" bollard = { version = "0.20", features = ["ssh"] } bytes = { workspace = true } futures = { workspace = true } -hmac = "0.12" miette = { workspace = true } rcgen = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -sha2 = "0.10" -url = { workspace = true } tar = "0.4" tempfile = "3" tokio = { workspace = true } diff --git a/crates/openshell-bootstrap/src/build.rs b/crates/openshell-bootstrap/src/build.rs index 3191b88b1..a313d4394 100644 --- a/crates/openshell-bootstrap/src/build.rs +++ b/crates/openshell-bootstrap/src/build.rs @@ -1,47 +1,25 @@ // SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -//! Build and export container images for gateway runtimes. +//! Build container images for gateway runtimes. //! //! This module wraps bollard's `build_image()` API to build a container image //! from a Dockerfile and build context. Kubernetes deployments reuse the //! existing push pipeline to import the image into the gateway's containerd -//! runtime, while the VM backend can export the built image as a rootfs tar. +//! runtime. VM deployments keep the built image in the local Docker daemon and +//! pass an internal local-image reference to the VM driver. use std::collections::HashMap; -use std::fs::File; -use std::io::Read; -use std::path::{Path, PathBuf}; +use std::path::Path; -use base64::Engine as _; -use base64::engine::general_purpose::URL_SAFE_NO_PAD; use bollard::Docker; -use bollard::models::ContainerCreateBody; -use bollard::query_parameters::{ - BuildImageOptionsBuilder, CreateContainerOptionsBuilder, RemoveContainerOptionsBuilder, -}; +use bollard::query_parameters::BuildImageOptionsBuilder; use futures::StreamExt; -use hmac::{Hmac, Mac}; use miette::{IntoDiagnostic, Result, WrapErr}; -use sha2::{Digest, Sha256}; -use tokio::io::AsyncWriteExt; -use url::{Position, Url}; use crate::constants::container_name; use crate::push::push_local_images; -/// Pseudo-image URI scheme used to hand a local rootfs tar artifact to the VM driver. -pub const ROOTFS_TAR_IMAGE_REF_SCHEME: &str = "openshell-rootfs-tar"; -const ROOTFS_TAR_IMAGE_REF_VERSION: &str = "v1"; -type HmacSha256 = Hmac; - -/// Authenticated VM rootfs tar artifact decoded from an internal image reference. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RootfsTarImageRef { - pub path: PathBuf, - pub digest: String, -} - /// Build a container image from a Dockerfile using the local Docker daemon. /// /// This is used by `openshell sandbox create --from ` for both the @@ -65,191 +43,6 @@ pub async fn build_local_image( Ok(()) } -/// Encode a local rootfs tar path as an internal image reference understood by the VM driver. -pub fn encode_rootfs_tar_image_ref(path: &Path) -> Result { - let canonical = path - .canonicalize() - .into_diagnostic() - .wrap_err_with(|| format!("failed to resolve rootfs tar path {}", path.display()))?; - let file_url = Url::from_file_path(&canonical).map_err(|()| { - miette::miette!("failed to encode rootfs tar path {}", canonical.display()) - })?; - Ok(format!( - "{ROOTFS_TAR_IMAGE_REF_SCHEME}:{}", - &file_url[Position::BeforePath..] - )) -} - -/// Encode a local rootfs tar path as an authenticated VM-driver artifact reference. -pub fn encode_authenticated_rootfs_tar_image_ref(path: &Path, secret: &str) -> Result { - let canonical = path - .canonicalize() - .into_diagnostic() - .wrap_err_with(|| format!("failed to resolve rootfs tar path {}", path.display()))?; - let digest = compute_file_sha256_hex(&canonical)?; - let file_url = Url::from_file_path(&canonical).map_err(|()| { - miette::miette!("failed to encode rootfs tar path {}", canonical.display()) - })?; - let file_url = file_url.to_string(); - let signature = sign_rootfs_tar_image_ref(secret, &file_url, &digest)?; - Ok(format!( - "{ROOTFS_TAR_IMAGE_REF_SCHEME}:{ROOTFS_TAR_IMAGE_REF_VERSION}:{}:{digest}:{}", - URL_SAFE_NO_PAD.encode(file_url.as_bytes()), - URL_SAFE_NO_PAD.encode(signature), - )) -} - -/// Decode a VM-driver rootfs tar image reference back to a local filesystem path. -pub fn decode_rootfs_tar_image_ref(image_ref: &str) -> Option { - let remainder = image_ref.strip_prefix(&format!("{ROOTFS_TAR_IMAGE_REF_SCHEME}:"))?; - let file_url = format!("file:{remainder}"); - Url::parse(&file_url).ok()?.to_file_path().ok() -} - -/// Decode and verify an authenticated VM-driver rootfs tar image reference. -pub fn decode_authenticated_rootfs_tar_image_ref( - image_ref: &str, - secret: &str, -) -> Result> { - let Some(remainder) = image_ref.strip_prefix(&format!( - "{ROOTFS_TAR_IMAGE_REF_SCHEME}:{ROOTFS_TAR_IMAGE_REF_VERSION}:" - )) else { - return Ok(None); - }; - - let mut parts = remainder.split(':'); - let encoded_url = parts - .next() - .filter(|value| !value.is_empty()) - .ok_or_else(|| miette::miette!("missing rootfs artifact path"))?; - let digest = parts - .next() - .filter(|value| !value.is_empty()) - .ok_or_else(|| miette::miette!("missing rootfs artifact digest"))?; - let encoded_signature = parts - .next() - .filter(|value| !value.is_empty()) - .ok_or_else(|| miette::miette!("missing rootfs artifact signature"))?; - if parts.next().is_some() { - return Err(miette::miette!("malformed rootfs artifact reference")); - } - validate_sha256_hex(digest)?; - - let file_url_bytes = URL_SAFE_NO_PAD - .decode(encoded_url) - .map_err(|err| miette::miette!("invalid rootfs artifact path encoding: {err}"))?; - let file_url = String::from_utf8(file_url_bytes) - .map_err(|err| miette::miette!("invalid rootfs artifact path encoding: {err}"))?; - - let signature = URL_SAFE_NO_PAD - .decode(encoded_signature) - .map_err(|err| miette::miette!("invalid rootfs artifact signature encoding: {err}"))?; - verify_rootfs_tar_image_ref_signature(secret, &file_url, digest, &signature)?; - - let url = Url::parse(&file_url) - .into_diagnostic() - .wrap_err("invalid rootfs artifact URL")?; - if url.scheme() != "file" { - return Err(miette::miette!("rootfs artifact URL must use file scheme")); - } - let path = url - .to_file_path() - .map_err(|()| miette::miette!("rootfs artifact URL is not a local file path"))?; - - Ok(Some(RootfsTarImageRef { - path, - digest: format!("sha256:{digest}"), - })) -} - -/// Export a locally-built Docker image as a persistent rootfs tar artifact for the VM driver. -pub async fn export_local_image_rootfs( - image_ref: &str, - on_log: &mut impl FnMut(String), -) -> Result { - let temp = tempfile::Builder::new() - .prefix("openshell-vm-rootfs-") - .suffix(".tar") - .tempfile() - .into_diagnostic() - .wrap_err("failed to allocate temporary VM rootfs artifact")?; - let temp_path = temp.path().to_path_buf(); - let (_file, output_path) = temp.keep().into_diagnostic().wrap_err_with(|| { - format!( - "failed to persist temporary VM rootfs artifact {}", - temp_path.display() - ) - })?; - - on_log(format!( - "Exporting built image {image_ref} as VM rootfs artifact {}", - output_path.display() - )); - export_local_image_rootfs_to_path(image_ref, &output_path).await?; - on_log(format!( - "Exported VM rootfs artifact {}", - output_path.display() - )); - Ok(output_path) -} - -fn compute_file_sha256_hex(path: &Path) -> Result { - let mut file = File::open(path) - .into_diagnostic() - .wrap_err_with(|| format!("failed to open {}", path.display()))?; - let mut hasher = Sha256::new(); - let mut buf = [0u8; 64 * 1024]; - loop { - let read = file - .read(&mut buf) - .into_diagnostic() - .wrap_err_with(|| format!("failed to read {}", path.display()))?; - if read == 0 { - break; - } - hasher.update(&buf[..read]); - } - Ok(format!("{:x}", hasher.finalize())) -} - -fn validate_sha256_hex(value: &str) -> Result<()> { - if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) { - Ok(()) - } else { - Err(miette::miette!("rootfs artifact digest must be sha256 hex")) - } -} - -fn rootfs_tar_image_ref_message(file_url: &str, digest: &str) -> String { - format!("{ROOTFS_TAR_IMAGE_REF_VERSION}\n{file_url}\nsha256:{digest}") -} - -fn rootfs_tar_image_ref_hmac(secret: &str) -> Result { - if secret.trim().is_empty() { - return Err(miette::miette!("rootfs artifact secret is empty")); - } - HmacSha256::new_from_slice(secret.as_bytes()) - .map_err(|err| miette::miette!("invalid rootfs artifact secret: {err}")) -} - -fn sign_rootfs_tar_image_ref(secret: &str, file_url: &str, digest: &str) -> Result> { - let mut mac = rootfs_tar_image_ref_hmac(secret)?; - mac.update(rootfs_tar_image_ref_message(file_url, digest).as_bytes()); - Ok(mac.finalize().into_bytes().to_vec()) -} - -fn verify_rootfs_tar_image_ref_signature( - secret: &str, - file_url: &str, - digest: &str, - signature: &[u8], -) -> Result<()> { - let mut mac = rootfs_tar_image_ref_hmac(secret)?; - mac.update(rootfs_tar_image_ref_message(file_url, digest).as_bytes()); - mac.verify_slice(signature) - .map_err(|_| miette::miette!("rootfs artifact signature verification failed")) -} - /// Push a locally-built image into the gateway's containerd runtime. #[allow(clippy::implicit_hasher)] pub async fn push_image_into_gateway( @@ -360,79 +153,6 @@ async fn build_image( Ok(()) } -async fn export_local_image_rootfs_to_path(image_ref: &str, tar_path: &Path) -> Result<()> { - let docker = Docker::connect_with_local_defaults() - .into_diagnostic() - .wrap_err("failed to connect to local Docker daemon")?; - let container_name = format!( - "openshell-rootfs-export-{}", - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - ); - let create_options = CreateContainerOptionsBuilder::default() - .name(container_name.as_str()) - .build(); - let container = docker - .create_container( - Some(create_options), - ContainerCreateBody { - image: Some(image_ref.to_string()), - ..Default::default() - }, - ) - .await - .into_diagnostic() - .wrap_err_with(|| { - format!("failed to create temporary export container for image {image_ref}") - })?; - let container_id = container.id; - - let export_result = async { - if let Some(parent) = tar_path.parent() { - tokio::fs::create_dir_all(parent) - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", parent.display()))?; - } - let mut file = tokio::fs::File::create(tar_path) - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to create {}", tar_path.display()))?; - let mut stream = docker.export_container(&container_id); - while let Some(chunk) = stream.next().await { - let chunk = chunk - .into_diagnostic() - .wrap_err_with(|| format!("failed to export image {image_ref}"))?; - file.write_all(&chunk) - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to write {}", tar_path.display()))?; - } - file.flush() - .await - .into_diagnostic() - .wrap_err_with(|| format!("failed to flush {}", tar_path.display())) - } - .await; - - let cleanup_result = docker - .remove_container( - &container_id, - Some(RemoveContainerOptionsBuilder::default().force(true).build()), - ) - .await; - - match (export_result, cleanup_result) { - (Ok(()), Ok(())) => Ok(()), - (Err(err), _) => Err(err), - (Ok(()), Err(err)) => Err(err).into_diagnostic().wrap_err_with(|| { - format!("failed to remove temporary export container for {image_ref}") - }), - } -} - /// Create a tar archive of a directory for use as a Docker build context. /// /// Walks `context_dir` recursively, respects a `.dockerignore` file if present, @@ -774,47 +494,4 @@ mod tests { assert!(is_ignored("node_modules", true, &patterns)); assert!(is_ignored("node_modules/foo.js", false, &patterns)); } - - #[test] - fn encode_and_decode_rootfs_tar_image_ref_round_trip() { - let dir = tempfile::tempdir().unwrap(); - let tar_path = dir.path().join("rootfs tar.tar"); - fs::write(&tar_path, "rootfs").unwrap(); - - let encoded = encode_rootfs_tar_image_ref(&tar_path).unwrap(); - let decoded = decode_rootfs_tar_image_ref(&encoded).unwrap(); - - assert_eq!(decoded, tar_path.canonicalize().unwrap()); - } - - #[test] - fn authenticated_rootfs_tar_image_ref_round_trip() { - let dir = tempfile::tempdir().unwrap(); - let tar_path = dir.path().join("rootfs tar.tar"); - fs::write(&tar_path, "rootfs").unwrap(); - - let encoded = encode_authenticated_rootfs_tar_image_ref(&tar_path, "secret").unwrap(); - let decoded = decode_authenticated_rootfs_tar_image_ref(&encoded, "secret") - .unwrap() - .unwrap(); - - assert_eq!(decoded.path, tar_path.canonicalize().unwrap()); - assert_eq!( - decoded.digest, - "sha256:3c47ef972d531d524daa15fa33dd885dd23de6221bbd10a29eb42ecfcf2ef422" - ); - } - - #[test] - fn authenticated_rootfs_tar_image_ref_rejects_wrong_secret() { - let dir = tempfile::tempdir().unwrap(); - let tar_path = dir.path().join("rootfs.tar"); - fs::write(&tar_path, "rootfs").unwrap(); - - let encoded = encode_authenticated_rootfs_tar_image_ref(&tar_path, "secret").unwrap(); - let err = decode_authenticated_rootfs_tar_image_ref(&encoded, "wrong-secret") - .expect_err("wrong secret should fail"); - - assert!(err.to_string().contains("signature verification failed")); - } } diff --git a/crates/openshell-bootstrap/src/metadata.rs b/crates/openshell-bootstrap/src/metadata.rs index 5892dd15f..ee6509e93 100644 --- a/crates/openshell-bootstrap/src/metadata.rs +++ b/crates/openshell-bootstrap/src/metadata.rs @@ -50,10 +50,6 @@ pub struct GatewayMetadata { /// Local VM driver state directory for standalone VM gateways. #[serde(default, skip_serializing_if = "Option::is_none")] pub vm_driver_state_dir: Option, - - /// Local secret used to authenticate VM rootfs artifact references. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub vm_rootfs_artifact_secret: Option, } impl GatewayMetadata { @@ -145,7 +141,6 @@ pub fn create_gateway_metadata_with_host( edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, } } @@ -474,7 +469,6 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }; let json = serde_json::to_string(&meta).unwrap(); let parsed: GatewayMetadata = serde_json::from_str(&json).unwrap(); @@ -571,7 +565,6 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }; assert_eq!(meta.gateway_host(), None); } @@ -589,7 +582,6 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }; assert_eq!(meta.gateway_host(), Some("10.0.0.5")); } diff --git a/crates/openshell-cli/src/completers.rs b/crates/openshell-cli/src/completers.rs index 31a98158c..3b87e4b88 100644 --- a/crates/openshell-cli/src/completers.rs +++ b/crates/openshell-cli/src/completers.rs @@ -184,7 +184,6 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }, ) .unwrap(); diff --git a/crates/openshell-cli/src/main.rs b/crates/openshell-cli/src/main.rs index da7b3d13a..7a8c93a36 100644 --- a/crates/openshell-cli/src/main.rs +++ b/crates/openshell-cli/src/main.rs @@ -2846,7 +2846,6 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, } } diff --git a/crates/openshell-cli/src/run.rs b/crates/openshell-cli/src/run.rs index edb3da982..ee8ea0c36 100644 --- a/crates/openshell-cli/src/run.rs +++ b/crates/openshell-cli/src/run.rs @@ -898,7 +898,6 @@ fn plaintext_gateway_metadata( edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, } } @@ -1101,7 +1100,6 @@ pub async fn gateway_add( edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }; store_gateway_metadata(name, &metadata)?; @@ -1132,7 +1130,6 @@ pub async fn gateway_add( edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }; store_gateway_metadata(name, &metadata)?; @@ -2663,6 +2660,12 @@ fn image_requests_gpu(image: &str) -> bool { image_name.contains("gpu") } +const VM_LOCAL_IMAGE_REF_SCHEME: &str = "openshell-vm-local-image"; + +fn vm_local_image_ref(image_ref: &str) -> String { + format!("{VM_LOCAL_IMAGE_REF_SCHEME}:{image_ref}") +} + fn dockerfile_sources_supported_for_gateway(metadata: Option<&GatewayMetadata>) -> bool { !metadata.is_some_and(|metadata| metadata.is_remote) } @@ -2671,8 +2674,9 @@ fn dockerfile_sources_supported_for_gateway(metadata: Option<&GatewayMetadata>) /// /// For local Kubernetes gateways running in Docker, this imports the built image /// into the gateway runtime and returns the Docker tag. For local VM gateways, -/// this exports the built image as a rootfs tar artifact and returns an internal -/// pseudo-image URI understood by the VM driver. +/// this returns an internal local-image URI. The VM driver resolves that URI +/// against the local Docker daemon and prepares the VM rootfs on the gateway +/// host. async fn build_from_dockerfile( dockerfile: &Path, context: &Path, @@ -2732,34 +2736,18 @@ async fn build_from_dockerfile( return Ok(tag); } - let rootfs_tar = openshell_bootstrap::build::export_local_image_rootfs(&tag, &mut on_log) - .await - .wrap_err("failed to export built image as a VM rootfs artifact")?; - let artifact_secret = metadata - .as_ref() - .and_then(|metadata| metadata.vm_rootfs_artifact_secret.as_deref()) - .filter(|secret| !secret.trim().is_empty()) - .ok_or_else(|| { - miette!( - "local Dockerfile sources for VM gateways require authenticated rootfs artifact metadata; restart gateway '{}' with a current `mise run gateway:vm`", - gateway_name - ) - })?; - let artifact_ref = openshell_bootstrap::build::encode_authenticated_rootfs_tar_image_ref( - &rootfs_tar, - artifact_secret, - )?; + let local_image_ref = vm_local_image_ref(&tag); eprintln!(); eprintln!( - "{} VM rootfs artifact {} is ready for gateway '{}'.", + "{} Image {} will be resolved by the local VM driver for gateway '{}'.", "✓".green().bold(), - rootfs_tar.display().to_string().cyan(), + tag.cyan(), gateway_name, ); eprintln!(); - Ok(artifact_ref) + Ok(local_image_ref) } /// Load sandbox policy YAML. @@ -5583,6 +5571,7 @@ mod tests { parse_credential_pairs, plaintext_gateway_is_remote, provisioning_timeout_message, ready_false_condition_message, resolve_gateway_control_target_from, sandbox_should_persist, shell_escape, source_requests_gpu, validate_gateway_name, validate_ssh_host, + vm_local_image_ref, }; use crate::TEST_ENV_LOCK; use hyper::StatusCode; @@ -5660,7 +5649,6 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, } } @@ -5847,7 +5835,6 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }; assert!(!dockerfile_sources_supported_for_gateway(Some(&metadata))); @@ -5866,13 +5853,20 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }; assert!(dockerfile_sources_supported_for_gateway(Some(&metadata))); assert!(dockerfile_sources_supported_for_gateway(None)); } + #[test] + fn vm_local_image_ref_wraps_docker_image_ref() { + assert_eq!( + vm_local_image_ref("openshell/sandbox-from:123"), + "openshell-vm-local-image:openshell/sandbox-from:123" + ); + } + #[test] fn ready_false_condition_message_prefers_reason_and_message() { let status = SandboxStatus { @@ -6089,7 +6083,6 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }, ]; @@ -6126,7 +6119,6 @@ mod tests { edge_team_domain: None, edge_auth_url: None, vm_driver_state_dir: None, - vm_rootfs_artifact_secret: None, }; assert_eq!(gateway_auth_label(&gateway), "mtls"); diff --git a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs index e69d06f4f..687ee87b2 100644 --- a/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs +++ b/crates/openshell-cli/tests/sandbox_create_lifecycle_integration.rs @@ -735,6 +735,10 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() { let _env = test_env(&fake_ssh_dir, &xdg_dir); let tls = test_tls(&server); install_fake_ssh(&fake_ssh_dir); + let forward_port = { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + listener.local_addr().unwrap().port() + }; run::sandbox_create( &server.endpoint, @@ -750,7 +754,7 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() { None, &[], None, - Some(openshell_core::forward::ForwardSpec::new(8080)), + Some(openshell_core::forward::ForwardSpec::new(forward_port)), &["echo".to_string(), "OK".to_string()], Some(false), Some(false), diff --git a/crates/openshell-driver-vm/Cargo.toml b/crates/openshell-driver-vm/Cargo.toml index 97c15d261..c13d904a6 100644 --- a/crates/openshell-driver-vm/Cargo.toml +++ b/crates/openshell-driver-vm/Cargo.toml @@ -21,8 +21,8 @@ path = "src/main.rs" [dependencies] openshell-core = { path = "../openshell-core" } openshell-vfio = { path = "../openshell-vfio" } -openshell-bootstrap = { path = "../openshell-bootstrap" } +bollard = { version = "0.20", features = ["ssh"] } tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } prost-types = { workspace = true } diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index d42dea442..49807332e 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -164,7 +164,9 @@ The VM guest's serial console is appended to `//console.l - Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest) - `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary. - [mise](https://mise.jdx.dev/) task runner -- Docker-compatible socket on the CLI host when using `openshell sandbox create --from ./Dockerfile` or `--from ./dir` +- Docker-compatible socket on the local CLI/gateway host when using + `openshell sandbox create --from ./Dockerfile` or `--from ./dir`; the CLI + builds the image and the VM driver exports it via the local Docker daemon - `gh` CLI (used by `mise run vm:setup` to download pre-built runtime artifacts) ## Relationship to `openshell-vm` diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 0270c5a5d..9dbd59808 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -8,8 +8,11 @@ use crate::rootfs::{ create_rootfs_archive_from_dir, extract_rootfs_archive_to, prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path, }; +use bollard::Docker; +use bollard::models::ContainerCreateBody; +use bollard::query_parameters::{CreateContainerOptionsBuilder, RemoveContainerOptionsBuilder}; use flate2::read::GzDecoder; -use futures::Stream; +use futures::{Stream, StreamExt}; use nix::errno::Errno; use nix::sys::signal::{Signal, kill}; use nix::unistd::Pid; @@ -17,9 +20,6 @@ use oci_client::client::{Client as OciClient, ClientConfig}; use oci_client::manifest::{ImageIndexEntry, OciDescriptor}; use oci_client::secrets::RegistryAuth; use oci_client::{Reference, RegistryOperation}; -use openshell_bootstrap::build::{ - ROOTFS_TAR_IMAGE_REF_SCHEME, decode_authenticated_rootfs_tar_image_ref, -}; use openshell_core::proto::compute::v1::{ CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse, DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent, @@ -89,8 +89,10 @@ const GUEST_TLS_CERT_PATH: &str = "/opt/openshell/tls/tls.crt"; const GUEST_TLS_KEY_PATH: &str = "/opt/openshell/tls/tls.key"; const IMAGE_CACHE_ROOT_DIR: &str = "images"; const IMAGE_CACHE_ROOTFS_ARCHIVE: &str = "rootfs.tar"; +const IMAGE_EXPORT_ROOTFS_ARCHIVE: &str = "source-rootfs.tar"; const IMAGE_IDENTITY_FILE: &str = "image-identity"; const IMAGE_REFERENCE_FILE: &str = "image-reference"; +const VM_LOCAL_IMAGE_REF_PREFIX: &str = "openshell-vm-local-image:"; static IMAGE_CACHE_BUILD_COUNTER: AtomicU64 = AtomicU64::new(0); #[derive(Debug, Clone)] @@ -118,7 +120,6 @@ pub struct VmDriverConfig { pub gpu_enabled: bool, pub gpu_mem_mib: u32, pub gpu_vcpus: u8, - pub rootfs_artifact_secret: Option, } impl Default for VmDriverConfig { @@ -140,7 +141,6 @@ impl Default for VmDriverConfig { gpu_enabled: false, gpu_mem_mib: 8192, gpu_vcpus: 4, - rootfs_artifact_secret: None, } } } @@ -760,32 +760,9 @@ impl VmDriver { sandbox_id: &str, image_ref: &str, ) -> Result { - if is_rootfs_tar_image_ref(image_ref) { - let secret = self - .config - .rootfs_artifact_secret - .as_deref() - .filter(|secret| !secret.trim().is_empty()) - .ok_or_else(|| { - Status::failed_precondition( - "vm rootfs tar image references require a gateway-issued artifact secret", - ) - })?; - let artifact = decode_authenticated_rootfs_tar_image_ref(image_ref, secret) - .map_err(|err| { - Status::failed_precondition(format!( - "invalid vm rootfs artifact reference: {err}" - )) - })? - .ok_or_else(|| { - Status::failed_precondition("invalid vm rootfs artifact reference") - })?; + if let Some(local_image_ref) = parse_vm_local_image_ref(image_ref)? { return self - .ensure_cached_rootfs_tar_image_rootfs_archive( - image_ref, - &artifact.path, - &artifact.digest, - ) + .ensure_cached_local_image_rootfs_archive(sandbox_id, image_ref, local_image_ref) .await; } @@ -876,58 +853,60 @@ impl VmDriver { Ok(image_identity) } - async fn ensure_cached_rootfs_tar_image_rootfs_archive( + async fn ensure_cached_local_image_rootfs_archive( &self, + sandbox_id: &str, image_ref: &str, - rootfs_tar_path: &Path, - expected_digest: &str, + local_image_ref: &str, ) -> Result { - let rootfs_tar = rootfs_tar_path.to_path_buf(); - let image_identity = tokio::task::spawn_blocking(move || compute_file_sha256(&rootfs_tar)) - .await - .map_err(|err| { - Status::internal(format!("rootfs tar digest computation panicked: {err}")) - })? - .map_err(|err| { - Status::failed_precondition(format!( - "failed to fingerprint vm sandbox rootfs artifact '{}': {err}", - rootfs_tar_path.display() - )) - })?; - if image_identity != expected_digest { - return Err(Status::failed_precondition( - "vm rootfs artifact digest does not match the authenticated reference", - )); - } + let docker = Docker::connect_with_local_defaults().map_err(|err| { + Status::failed_precondition(format!( + "failed to connect to local Docker daemon for vm local image '{image_ref}': {err}" + )) + })?; + let image_identity = local_docker_image_identity(&docker, local_image_ref).await?; let archive_path = image_cache_rootfs_archive(&self.config.state_dir, &image_identity); + self.publish_platform_event( + sandbox_id.to_string(), + platform_event( + "vm", + "Normal", + "Pulling", + format!("Pulling image \"{local_image_ref}\""), + ), + ); + if tokio::fs::metadata(&archive_path).await.is_ok() { + self.publish_pulled_event(sandbox_id, local_image_ref, &archive_path) + .await; return Ok(image_identity); } let _cache_guard = self.image_cache_lock.lock().await; if tokio::fs::metadata(&archive_path).await.is_ok() { + self.publish_pulled_event(sandbox_id, local_image_ref, &archive_path) + .await; return Ok(image_identity); } - self.build_cached_rootfs_tar_image_rootfs_archive( - image_ref, - rootfs_tar_path, - &image_identity, - ) - .await?; + self.build_cached_local_image_rootfs_archive(&docker, local_image_ref, &image_identity) + .await?; + self.publish_pulled_event(sandbox_id, local_image_ref, &archive_path) + .await; Ok(image_identity) } - async fn build_cached_rootfs_tar_image_rootfs_archive( + async fn build_cached_local_image_rootfs_archive( &self, + docker: &Docker, image_ref: &str, - rootfs_tar_path: &Path, image_identity: &str, ) -> Result<(), Status> { let cache_dir = image_cache_dir(&self.config.state_dir, image_identity); let archive_path = image_cache_rootfs_archive(&self.config.state_dir, image_identity); let staging_dir = image_cache_staging_dir(&self.config.state_dir, image_identity); + let exported_rootfs = staging_dir.join(IMAGE_EXPORT_ROOTFS_ARCHIVE); let prepared_rootfs = staging_dir.join("rootfs"); let prepared_archive = staging_dir.join(IMAGE_CACHE_ROOTFS_ARCHIVE); @@ -953,24 +932,29 @@ impl VmDriver { Status::internal(format!("create image cache staging dir failed: {err}")) })?; + if let Err(err) = + export_local_image_rootfs_to_path(docker, image_ref, &exported_rootfs).await + { + let _ = tokio::fs::remove_dir_all(&staging_dir).await; + return Err(err); + } + let image_ref_owned = image_ref.to_string(); let image_identity_owned = image_identity.to_string(); - let rootfs_tar_path_owned = rootfs_tar_path.to_path_buf(); + let exported_rootfs_for_build = exported_rootfs.clone(); let prepared_rootfs_for_build = prepared_rootfs.clone(); let prepared_archive_for_build = prepared_archive.clone(); let build_result = tokio::task::spawn_blocking(move || { - extract_rootfs_archive_to(&rootfs_tar_path_owned, &prepared_rootfs_for_build)?; - prepare_sandbox_rootfs_from_image_root( - &prepared_rootfs_for_build, + prepare_exported_rootfs_archive( + &image_ref_owned, &image_identity_owned, + &exported_rootfs_for_build, + &prepared_rootfs_for_build, + &prepared_archive_for_build, ) - .map_err(|err| { - format!("vm sandbox image '{image_ref_owned}' is not base-compatible: {err}") - })?; - create_rootfs_archive_from_dir(&prepared_rootfs_for_build, &prepared_archive_for_build) }) .await - .map_err(|err| Status::internal(format!("rootfs artifact preparation panicked: {err}")))?; + .map_err(|err| Status::internal(format!("local image preparation panicked: {err}")))?; if let Err(err) = build_result { let _ = tokio::fs::remove_dir_all(&staging_dir).await; @@ -1436,10 +1420,122 @@ fn parse_registry_reference(image_ref: &str) -> Result { }) } -fn is_rootfs_tar_image_ref(image_ref: &str) -> bool { - image_ref - .strip_prefix(ROOTFS_TAR_IMAGE_REF_SCHEME) - .is_some_and(|rest| rest.starts_with(':')) +#[allow(clippy::result_large_err)] +fn parse_vm_local_image_ref(image_ref: &str) -> Result, Status> { + let Some(local_image_ref) = image_ref.strip_prefix(VM_LOCAL_IMAGE_REF_PREFIX) else { + return Ok(None); + }; + if local_image_ref.trim().is_empty() { + return Err(Status::failed_precondition( + "invalid vm local image reference: missing Docker image reference", + )); + } + Ok(Some(local_image_ref)) +} + +async fn local_docker_image_identity(docker: &Docker, image_ref: &str) -> Result { + let inspect = docker.inspect_image(image_ref).await.map_err(|err| { + Status::failed_precondition(format!( + "failed to inspect local Docker image '{image_ref}': {err}" + )) + })?; + inspect + .id + .filter(|id| !id.trim().is_empty()) + .ok_or_else(|| { + Status::failed_precondition(format!( + "local Docker image '{image_ref}' inspect response has no image ID" + )) + }) +} + +async fn export_local_image_rootfs_to_path( + docker: &Docker, + image_ref: &str, + tar_path: &Path, +) -> Result<(), Status> { + let container_name = format!( + "openshell-vm-rootfs-export-{}", + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + ); + let create_options = CreateContainerOptionsBuilder::default() + .name(container_name.as_str()) + .build(); + let container = docker + .create_container( + Some(create_options), + ContainerCreateBody { + image: Some(image_ref.to_string()), + ..Default::default() + }, + ) + .await + .map_err(|err| { + Status::failed_precondition(format!( + "failed to create temporary export container for local Docker image '{image_ref}': {err}" + )) + })?; + let container_id = container.id; + + let export_result = async { + if let Some(parent) = tar_path.parent() { + tokio::fs::create_dir_all(parent).await.map_err(|err| { + Status::internal(format!( + "create export dir {} failed: {err}", + parent.display() + )) + })?; + } + let mut file = tokio::fs::File::create(tar_path).await.map_err(|err| { + Status::internal(format!("create {} failed: {err}", tar_path.display())) + })?; + let mut stream = docker.export_container(&container_id); + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|err| { + Status::failed_precondition(format!( + "failed to export local Docker image '{image_ref}': {err}" + )) + })?; + file.write_all(&chunk).await.map_err(|err| { + Status::internal(format!("write {} failed: {err}", tar_path.display())) + })?; + } + file.flush() + .await + .map_err(|err| Status::internal(format!("flush {} failed: {err}", tar_path.display()))) + } + .await; + + let cleanup_result = docker + .remove_container( + &container_id, + Some(RemoveContainerOptionsBuilder::default().force(true).build()), + ) + .await; + + match (export_result, cleanup_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(err), _) => Err(err), + (Ok(()), Err(err)) => Err(Status::internal(format!( + "failed to remove temporary export container for local Docker image '{image_ref}': {err}" + ))), + } +} + +fn prepare_exported_rootfs_archive( + image_ref: &str, + image_identity: &str, + exported_rootfs: &Path, + prepared_rootfs: &Path, + prepared_archive: &Path, +) -> Result<(), String> { + extract_rootfs_archive_to(exported_rootfs, prepared_rootfs)?; + prepare_sandbox_rootfs_from_image_root(prepared_rootfs, image_identity) + .map_err(|err| format!("vm sandbox image '{image_ref}' is not base-compatible: {err}"))?; + create_rootfs_archive_from_dir(prepared_rootfs, prepared_archive) } fn registry_client() -> OciClient { @@ -1700,10 +1796,6 @@ fn verify_descriptor_digest(path: &Path, expected_digest: &str) -> Result<(), St } } -fn compute_file_sha256(path: &Path) -> Result { - compute_file_sha256_hex(path).map(|digest| format!("sha256:{digest}")) -} - fn compute_file_sha256_hex(path: &Path) -> Result { let mut file = fs::File::open(path).map_err(|err| format!("open {}: {err}", path.display()))?; let mut hasher = Sha256::new(); @@ -2585,29 +2677,28 @@ mod tests { ); } - #[tokio::test] - async fn ensure_cached_image_rootfs_archive_rejects_unsigned_rootfs_tar_refs() { - let base = unique_temp_dir(); - fs::create_dir_all(&base).unwrap(); - let artifact = base.join("rootfs.tar"); - fs::write(&artifact, "not a real rootfs").unwrap(); - let image_ref = openshell_bootstrap::build::encode_rootfs_tar_image_ref(&artifact).unwrap(); - let driver = test_driver(VmDriverConfig { - state_dir: base.join("driver-state"), - ..Default::default() - }); - - let err = driver - .ensure_cached_image_rootfs_archive("sandbox-123", &image_ref) - .await - .expect_err("unsigned rootfs tar refs must be rejected"); - - assert_eq!(err.code(), Code::FailedPrecondition); - assert!( - err.message().contains("gateway-issued artifact secret"), - "unexpected error: {}", - err.message() + #[test] + fn parse_vm_local_image_ref_classifies_prefixed_refs() { + assert_eq!( + parse_vm_local_image_ref("openshell-vm-local-image:openshell/sandbox-from:123") + .unwrap(), + Some("openshell/sandbox-from:123") ); + assert_eq!(parse_vm_local_image_ref("ubuntu:24.04").unwrap(), None); + } + + #[test] + fn parse_vm_local_image_ref_rejects_empty_refs() { + for image_ref in ["openshell-vm-local-image:", "openshell-vm-local-image: "] { + let err = + parse_vm_local_image_ref(image_ref).expect_err("empty local image refs must fail"); + assert_eq!(err.code(), Code::FailedPrecondition); + assert!( + err.message().contains("missing Docker image reference"), + "unexpected error: {}", + err.message() + ); + } } #[test] @@ -2778,15 +2869,56 @@ mod tests { } #[test] - fn compute_file_sha256_returns_prefixed_digest() { + fn prepare_exported_rootfs_archive_rewrites_docker_exported_rootfs() { let base = unique_temp_dir(); - fs::create_dir_all(&base).unwrap(); - let file = base.join("rootfs.tar"); - fs::write(&file, b"openshell").unwrap(); + let source_rootfs = base.join("source-rootfs"); + let exported_rootfs = base.join("exported-rootfs.tar"); + let prepared_rootfs = base.join("prepared-rootfs"); + let prepared_archive = base.join("prepared-rootfs.tar"); + let extracted = base.join("extracted"); + for path in [ + "bin/bash", + "bin/mount", + "bin/sed", + "sbin/ip", + "opt/openshell/bin/openshell-sandbox", + "usr/local/bin/k3s", + ] { + let path = source_rootfs.join(path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, "").unwrap(); + } + fs::create_dir_all(source_rootfs.join("opt/openshell/manifests")).unwrap(); + fs::write(source_rootfs.join("opt/openshell/manifests/old.yaml"), "").unwrap(); + + create_rootfs_archive_from_dir(&source_rootfs, &exported_rootfs).unwrap(); + prepare_exported_rootfs_archive( + "openshell/sandbox-from:123", + "sha256:local-image", + &exported_rootfs, + &prepared_rootfs, + &prepared_archive, + ) + .unwrap(); + extract_rootfs_archive_to(&prepared_archive, &extracted).unwrap(); + + assert!(extracted.join("srv/openshell-vm-sandbox-init.sh").is_file()); + assert!( + extracted + .join("opt/openshell/bin/openshell-sandbox") + .is_file() + ); + assert!(!extracted.join("usr/local/bin/k3s").exists()); + assert!(!extracted.join("opt/openshell/manifests").exists()); assert_eq!( - compute_file_sha256(&file).unwrap(), - "sha256:dc5cbc21a452a783ec453e8a8603101dfec5c7d6a19b6c645889bec8b97c2390" + fs::read_to_string(extracted.join("opt/openshell/.rootfs-type")).unwrap(), + "sandbox\n" + ); + assert!( + fs::read_to_string(extracted.join(".openshell-rootfs-variant")) + .unwrap() + .contains("sha256:local-image") ); let _ = fs::remove_dir_all(base); @@ -2899,21 +3031,6 @@ mod tests { )) } - fn test_driver(config: VmDriverConfig) -> VmDriver { - VmDriver { - config, - launcher_bin: PathBuf::from("openshell-driver-vm"), - registry: Arc::new(Mutex::new(HashMap::new())), - image_cache_lock: Arc::new(Mutex::new(())), - events: broadcast::channel(WATCH_BUFFER).0, - gpu_inventory: None, - subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new( - Ipv4Addr::new(10, 0, 128, 0), - 17, - ))), - } - } - fn spawn_exited_child() -> Child { Command::new("sh") .arg("-c") diff --git a/crates/openshell-driver-vm/src/main.rs b/crates/openshell-driver-vm/src/main.rs index 3f5d70c14..596e6c88d 100644 --- a/crates/openshell-driver-vm/src/main.rs +++ b/crates/openshell-driver-vm/src/main.rs @@ -105,9 +105,6 @@ struct Args { #[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)] gpu_vcpus: u8, - #[arg(long, env = "OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET", hide = true)] - rootfs_artifact_secret: Option, - #[arg(long, hide = true)] vm_backend: Option, @@ -188,7 +185,6 @@ async fn main() -> Result<()> { gpu_enabled: args.gpu, gpu_mem_mib: args.gpu_mem_mib, gpu_vcpus: args.gpu_vcpus, - rootfs_artifact_secret: args.rootfs_artifact_secret, }) .await .map_err(|err| miette::miette!("{err}"))?; diff --git a/crates/openshell-vm/src/lib.rs b/crates/openshell-vm/src/lib.rs index b4d8081c1..ff01c0900 100644 --- a/crates/openshell-vm/src/lib.rs +++ b/crates/openshell-vm/src/lib.rs @@ -1733,6 +1733,7 @@ fn bootstrap_gateway(rootfs: &Path, gateway_name: &str, gateway_port: u16) -> Re auth_mode: None, edge_team_domain: None, edge_auth_url: None, + vm_driver_state_dir: None, }; let exec_socket = vm_exec_socket_path(rootfs); diff --git a/docs/sandboxes/manage-sandboxes.mdx b/docs/sandboxes/manage-sandboxes.mdx index fb24bae9b..da7c2e7f4 100644 --- a/docs/sandboxes/manage-sandboxes.mdx +++ b/docs/sandboxes/manage-sandboxes.mdx @@ -51,6 +51,10 @@ openshell sandbox create --from my-registry.example.com/my-image:latest The CLI resolves community names against the [OpenShell Community](https://github.com/NVIDIA/OpenShell-Community) catalog, pulls the bundled Dockerfile and policy, builds the image locally, and creates the sandbox. For the full catalog and how to contribute your own, refer to [Community Sandboxes](/sandboxes/community-sandboxes). +Local directories and Dockerfiles require a local gateway because the CLI builds +through the local Docker daemon. Use a registry image reference for remote +gateways. + ### Label a Sandbox Attach labels when you create a sandbox to track ownership, environment, or workflow grouping: diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh index e67f61347..cdeaa7931 100755 --- a/tasks/scripts/gateway-vm.sh +++ b/tasks/scripts/gateway-vm.sh @@ -85,7 +85,6 @@ register_gateway_metadata() { local endpoint=$2 local port=$3 local vm_driver_state_dir=$4 - local rootfs_artifact_secret=$5 local config_home gateway_dir config_home="${XDG_CONFIG_HOME:-${HOME}/.config}" @@ -100,8 +99,7 @@ register_gateway_metadata() { "is_remote": false, "gateway_port": ${port}, "auth_mode": "plaintext", - "vm_driver_state_dir": "${vm_driver_state_dir}", - "vm_rootfs_artifact_secret": "${rootfs_artifact_secret}" + "vm_driver_state_dir": "${vm_driver_state_dir}" } EOF chmod 600 "${gateway_dir}/metadata.json" 2>/dev/null || true @@ -152,10 +150,6 @@ check_supervisor_cross_toolchain() { fi } -generate_rootfs_artifact_secret() { - od -An -N32 -tx1 /dev/urandom | tr -d ' \n' -} - VM_GPU="$(normalize_bool "${OPENSHELL_VM_GPU:-false}")" while [ "$#" -gt 0 ]; do @@ -219,12 +213,6 @@ VM_DRIVER_STATE_DIR_DEFAULT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}/openshell-v VM_DRIVER_STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${VM_DRIVER_STATE_DIR_DEFAULT}}" DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")" -ROOTFS_ARTIFACT_SECRET="${OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET:-$(generate_rootfs_artifact_secret)}" -if [[ ! "${ROOTFS_ARTIFACT_SECRET}" =~ ^[A-Za-z0-9._~=-]+$ ]]; then - echo "ERROR: OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET must contain only URL-safe characters" >&2 - exit 2 -fi -export OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET="${ROOTFS_ARTIFACT_SECRET}" # Build prerequisites: VM runtime artifacts + bundled supervisor. if [ ! -d "${COMPRESSED_DIR}" ] \ @@ -264,7 +252,7 @@ mkdir -p "${STATE_DIR}" mkdir -p "${VM_DRIVER_STATE_DIR}" GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" -register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}" "${ROOTFS_ARTIFACT_SECRET}" +register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}" save_active_gateway "${GATEWAY_NAME}" echo "Starting standalone VM gateway..."