mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-19 05:50:41 +08:00
refactor(vm): move local image handling into driver
This commit is contained in:
Generated
+1
-4
@@ -3278,18 +3278,15 @@ dependencies = [
|
||||
"bollard",
|
||||
"bytes",
|
||||
"futures",
|
||||
"hmac",
|
||||
"miette",
|
||||
"openshell-core",
|
||||
"rcgen",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"tar",
|
||||
"tempfile",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3422,6 +3419,7 @@ dependencies = [
|
||||
name = "openshell-driver-vm"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"bollard",
|
||||
"clap",
|
||||
"flate2",
|
||||
"futures",
|
||||
@@ -3430,7 +3428,6 @@ dependencies = [
|
||||
"miette",
|
||||
"nix",
|
||||
"oci-client",
|
||||
"openshell-bootstrap",
|
||||
"openshell-core",
|
||||
"openshell-vfio",
|
||||
"polling",
|
||||
|
||||
@@ -76,9 +76,12 @@ Old runtime cache versions are cleaned up when a new version is extracted.
|
||||
### Sandbox rootfs preparation
|
||||
|
||||
Each VM sandbox starts from either a registry image fetched directly over OCI or
|
||||
a local rootfs tar artifact exported by the CLI for Dockerfile-based `--from`
|
||||
sources, then the driver **rewrites that filesystem into a supervisor-only
|
||||
sandbox guest** before caching it:
|
||||
a local Docker image reference produced by Dockerfile-based `--from` sources.
|
||||
For local Dockerfile sources, the CLI builds the image on the local Docker
|
||||
daemon and passes the VM driver an internal `openshell-vm-local-image:<tag>`
|
||||
reference. The driver resolves that tag on the gateway host, exports the image
|
||||
filesystem, and **rewrites that filesystem into a supervisor-only sandbox
|
||||
guest** before caching it:
|
||||
|
||||
- `/srv/openshell-vm-sandbox-init.sh` is installed as the guest entrypoint
|
||||
- the bundled `openshell-sandbox` binary is copied into
|
||||
@@ -108,7 +111,7 @@ sandbox image. The driver:
|
||||
- resolves the image on the gateway host without Docker for registry and
|
||||
community image refs
|
||||
- for local Dockerfile sources, the CLI builds through the host Docker socket
|
||||
and hands the VM driver a local rootfs tar artifact instead of a Docker tag
|
||||
and hands the VM driver an internal local-image ref instead of a registry ref
|
||||
- unpacks the image filesystem, injects the VM sandbox init/supervisor files,
|
||||
and validates required guest tools such as `bash`, `mount`, `ip`, and `sed`
|
||||
- caches the prepared guest rootfs under
|
||||
@@ -116,16 +119,20 @@ sandbox image. The driver:
|
||||
- extracts a private runtime copy under
|
||||
`<vm-driver-state-dir>/sandboxes/<sandbox-id>/rootfs`
|
||||
|
||||
The cache key uses an immutable image identity: repo digest when available,
|
||||
otherwise a SHA-256 fingerprint of the local rootfs tar artifact.
|
||||
The cache key uses an immutable image identity: repo digest for registry images
|
||||
and the local Docker image ID for local-image refs.
|
||||
Different VM sandboxes can use different base images concurrently because the
|
||||
shared cache is per image, not global for the driver. Cached prepared rootfs
|
||||
entries remain on disk until the operator removes them from the VM driver state
|
||||
directory.
|
||||
|
||||
Docker is therefore no longer required for VM sandboxes created from registry or
|
||||
community image refs. It is only required on the CLI host when the source is a
|
||||
local Dockerfile or build context.
|
||||
community image refs. It is only required on the local CLI/gateway host when the
|
||||
source is a local Dockerfile or build context.
|
||||
|
||||
Local Dockerfile sources are treated as trusted local-development inputs for VM
|
||||
gateways. Remote VM gateways still reject local Dockerfile sources until a
|
||||
gateway-side artifact validation and transfer boundary is designed.
|
||||
|
||||
There is no embedded guest rootfs fallback anymore. VM sandboxes therefore
|
||||
require either `template.image` or a configured default sandbox image. This is
|
||||
|
||||
@@ -9,7 +9,7 @@ The `--from` flag accepts four kinds of input:
|
||||
| Input | Example | Behavior |
|
||||
|-------|---------|----------|
|
||||
| **Community sandbox name** | `--from openclaw` | Resolves to `ghcr.io/nvidia/openshell-community/sandboxes/openclaw:latest` |
|
||||
| **Dockerfile path** | `--from ./Dockerfile` | Builds the image, pushes it into the cluster, then creates the sandbox |
|
||||
| **Dockerfile path** | `--from ./Dockerfile` | Builds the image locally, makes it available to the local gateway, then creates the sandbox |
|
||||
| **Directory with Dockerfile** | `--from ./my-sandbox/` | Uses the directory as the build context |
|
||||
| **Full image reference** | `--from myregistry.com/img:tag` | Uses the image directly |
|
||||
|
||||
@@ -34,8 +34,13 @@ When `--from` points to a Dockerfile or directory, the CLI:
|
||||
|
||||
1. Builds the image locally via the Docker daemon (respecting `.dockerignore`).
|
||||
2. For a local Kubernetes gateway, pushes it into the cluster's containerd runtime using `docker save` / `ctr import`.
|
||||
3. For a local VM gateway, exports the built image as a rootfs tar artifact and passes the VM driver a gateway-signed internal artifact reference. The driver rejects unsigned `openshell-rootfs-tar:` references so API clients cannot point it at arbitrary gateway host files.
|
||||
4. Creates the sandbox with the resulting image tag or VM artifact reference.
|
||||
3. For a local VM gateway, passes the VM driver an internal `openshell-vm-local-image:<tag>` reference. The driver resolves that tag against the local Docker daemon, exports the image filesystem, and prepares the VM rootfs in its own cache.
|
||||
4. Creates the sandbox with the resulting image tag or VM-local image reference.
|
||||
|
||||
Local Dockerfile sources for VM gateways are trusted local-development inputs.
|
||||
Remote gateways continue to reject local Dockerfile sources because the gateway
|
||||
API does not yet validate or transfer local build artifacts across that
|
||||
boundary.
|
||||
|
||||
## How It Works
|
||||
|
||||
|
||||
@@ -16,13 +16,10 @@ base64 = "0.22"
|
||||
bollard = { version = "0.20", features = ["ssh"] }
|
||||
bytes = { workspace = true }
|
||||
futures = { workspace = true }
|
||||
hmac = "0.12"
|
||||
miette = { workspace = true }
|
||||
rcgen = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = "0.10"
|
||||
url = { workspace = true }
|
||||
tar = "0.4"
|
||||
tempfile = "3"
|
||||
tokio = { workspace = true }
|
||||
|
||||
@@ -1,47 +1,25 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Build and export container images for gateway runtimes.
|
||||
//! Build container images for gateway runtimes.
|
||||
//!
|
||||
//! This module wraps bollard's `build_image()` API to build a container image
|
||||
//! from a Dockerfile and build context. Kubernetes deployments reuse the
|
||||
//! existing push pipeline to import the image into the gateway's containerd
|
||||
//! runtime, while the VM backend can export the built image as a rootfs tar.
|
||||
//! runtime. VM deployments keep the built image in the local Docker daemon and
|
||||
//! pass an internal local-image reference to the VM driver.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::path::Path;
|
||||
|
||||
use base64::Engine as _;
|
||||
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
|
||||
use bollard::Docker;
|
||||
use bollard::models::ContainerCreateBody;
|
||||
use bollard::query_parameters::{
|
||||
BuildImageOptionsBuilder, CreateContainerOptionsBuilder, RemoveContainerOptionsBuilder,
|
||||
};
|
||||
use bollard::query_parameters::BuildImageOptionsBuilder;
|
||||
use futures::StreamExt;
|
||||
use hmac::{Hmac, Mac};
|
||||
use miette::{IntoDiagnostic, Result, WrapErr};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use url::{Position, Url};
|
||||
|
||||
use crate::constants::container_name;
|
||||
use crate::push::push_local_images;
|
||||
|
||||
/// Pseudo-image URI scheme used to hand a local rootfs tar artifact to the VM driver.
|
||||
pub const ROOTFS_TAR_IMAGE_REF_SCHEME: &str = "openshell-rootfs-tar";
|
||||
const ROOTFS_TAR_IMAGE_REF_VERSION: &str = "v1";
|
||||
type HmacSha256 = Hmac<Sha256>;
|
||||
|
||||
/// Authenticated VM rootfs tar artifact decoded from an internal image reference.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct RootfsTarImageRef {
|
||||
pub path: PathBuf,
|
||||
pub digest: String,
|
||||
}
|
||||
|
||||
/// Build a container image from a Dockerfile using the local Docker daemon.
|
||||
///
|
||||
/// This is used by `openshell sandbox create --from <Dockerfile>` for both the
|
||||
@@ -65,191 +43,6 @@ pub async fn build_local_image(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Encode a local rootfs tar path as an internal image reference understood by the VM driver.
|
||||
pub fn encode_rootfs_tar_image_ref(path: &Path) -> Result<String> {
|
||||
let canonical = path
|
||||
.canonicalize()
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to resolve rootfs tar path {}", path.display()))?;
|
||||
let file_url = Url::from_file_path(&canonical).map_err(|()| {
|
||||
miette::miette!("failed to encode rootfs tar path {}", canonical.display())
|
||||
})?;
|
||||
Ok(format!(
|
||||
"{ROOTFS_TAR_IMAGE_REF_SCHEME}:{}",
|
||||
&file_url[Position::BeforePath..]
|
||||
))
|
||||
}
|
||||
|
||||
/// Encode a local rootfs tar path as an authenticated VM-driver artifact reference.
|
||||
pub fn encode_authenticated_rootfs_tar_image_ref(path: &Path, secret: &str) -> Result<String> {
|
||||
let canonical = path
|
||||
.canonicalize()
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to resolve rootfs tar path {}", path.display()))?;
|
||||
let digest = compute_file_sha256_hex(&canonical)?;
|
||||
let file_url = Url::from_file_path(&canonical).map_err(|()| {
|
||||
miette::miette!("failed to encode rootfs tar path {}", canonical.display())
|
||||
})?;
|
||||
let file_url = file_url.to_string();
|
||||
let signature = sign_rootfs_tar_image_ref(secret, &file_url, &digest)?;
|
||||
Ok(format!(
|
||||
"{ROOTFS_TAR_IMAGE_REF_SCHEME}:{ROOTFS_TAR_IMAGE_REF_VERSION}:{}:{digest}:{}",
|
||||
URL_SAFE_NO_PAD.encode(file_url.as_bytes()),
|
||||
URL_SAFE_NO_PAD.encode(signature),
|
||||
))
|
||||
}
|
||||
|
||||
/// Decode a VM-driver rootfs tar image reference back to a local filesystem path.
|
||||
pub fn decode_rootfs_tar_image_ref(image_ref: &str) -> Option<PathBuf> {
|
||||
let remainder = image_ref.strip_prefix(&format!("{ROOTFS_TAR_IMAGE_REF_SCHEME}:"))?;
|
||||
let file_url = format!("file:{remainder}");
|
||||
Url::parse(&file_url).ok()?.to_file_path().ok()
|
||||
}
|
||||
|
||||
/// Decode and verify an authenticated VM-driver rootfs tar image reference.
|
||||
pub fn decode_authenticated_rootfs_tar_image_ref(
|
||||
image_ref: &str,
|
||||
secret: &str,
|
||||
) -> Result<Option<RootfsTarImageRef>> {
|
||||
let Some(remainder) = image_ref.strip_prefix(&format!(
|
||||
"{ROOTFS_TAR_IMAGE_REF_SCHEME}:{ROOTFS_TAR_IMAGE_REF_VERSION}:"
|
||||
)) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
let mut parts = remainder.split(':');
|
||||
let encoded_url = parts
|
||||
.next()
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(|| miette::miette!("missing rootfs artifact path"))?;
|
||||
let digest = parts
|
||||
.next()
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(|| miette::miette!("missing rootfs artifact digest"))?;
|
||||
let encoded_signature = parts
|
||||
.next()
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or_else(|| miette::miette!("missing rootfs artifact signature"))?;
|
||||
if parts.next().is_some() {
|
||||
return Err(miette::miette!("malformed rootfs artifact reference"));
|
||||
}
|
||||
validate_sha256_hex(digest)?;
|
||||
|
||||
let file_url_bytes = URL_SAFE_NO_PAD
|
||||
.decode(encoded_url)
|
||||
.map_err(|err| miette::miette!("invalid rootfs artifact path encoding: {err}"))?;
|
||||
let file_url = String::from_utf8(file_url_bytes)
|
||||
.map_err(|err| miette::miette!("invalid rootfs artifact path encoding: {err}"))?;
|
||||
|
||||
let signature = URL_SAFE_NO_PAD
|
||||
.decode(encoded_signature)
|
||||
.map_err(|err| miette::miette!("invalid rootfs artifact signature encoding: {err}"))?;
|
||||
verify_rootfs_tar_image_ref_signature(secret, &file_url, digest, &signature)?;
|
||||
|
||||
let url = Url::parse(&file_url)
|
||||
.into_diagnostic()
|
||||
.wrap_err("invalid rootfs artifact URL")?;
|
||||
if url.scheme() != "file" {
|
||||
return Err(miette::miette!("rootfs artifact URL must use file scheme"));
|
||||
}
|
||||
let path = url
|
||||
.to_file_path()
|
||||
.map_err(|()| miette::miette!("rootfs artifact URL is not a local file path"))?;
|
||||
|
||||
Ok(Some(RootfsTarImageRef {
|
||||
path,
|
||||
digest: format!("sha256:{digest}"),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Export a locally-built Docker image as a persistent rootfs tar artifact for the VM driver.
|
||||
pub async fn export_local_image_rootfs(
|
||||
image_ref: &str,
|
||||
on_log: &mut impl FnMut(String),
|
||||
) -> Result<PathBuf> {
|
||||
let temp = tempfile::Builder::new()
|
||||
.prefix("openshell-vm-rootfs-")
|
||||
.suffix(".tar")
|
||||
.tempfile()
|
||||
.into_diagnostic()
|
||||
.wrap_err("failed to allocate temporary VM rootfs artifact")?;
|
||||
let temp_path = temp.path().to_path_buf();
|
||||
let (_file, output_path) = temp.keep().into_diagnostic().wrap_err_with(|| {
|
||||
format!(
|
||||
"failed to persist temporary VM rootfs artifact {}",
|
||||
temp_path.display()
|
||||
)
|
||||
})?;
|
||||
|
||||
on_log(format!(
|
||||
"Exporting built image {image_ref} as VM rootfs artifact {}",
|
||||
output_path.display()
|
||||
));
|
||||
export_local_image_rootfs_to_path(image_ref, &output_path).await?;
|
||||
on_log(format!(
|
||||
"Exported VM rootfs artifact {}",
|
||||
output_path.display()
|
||||
));
|
||||
Ok(output_path)
|
||||
}
|
||||
|
||||
fn compute_file_sha256_hex(path: &Path) -> Result<String> {
|
||||
let mut file = File::open(path)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to open {}", path.display()))?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut buf = [0u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file
|
||||
.read(&mut buf)
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to read {}", path.display()))?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..read]);
|
||||
}
|
||||
Ok(format!("{:x}", hasher.finalize()))
|
||||
}
|
||||
|
||||
fn validate_sha256_hex(value: &str) -> Result<()> {
|
||||
if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(miette::miette!("rootfs artifact digest must be sha256 hex"))
|
||||
}
|
||||
}
|
||||
|
||||
fn rootfs_tar_image_ref_message(file_url: &str, digest: &str) -> String {
|
||||
format!("{ROOTFS_TAR_IMAGE_REF_VERSION}\n{file_url}\nsha256:{digest}")
|
||||
}
|
||||
|
||||
fn rootfs_tar_image_ref_hmac(secret: &str) -> Result<HmacSha256> {
|
||||
if secret.trim().is_empty() {
|
||||
return Err(miette::miette!("rootfs artifact secret is empty"));
|
||||
}
|
||||
HmacSha256::new_from_slice(secret.as_bytes())
|
||||
.map_err(|err| miette::miette!("invalid rootfs artifact secret: {err}"))
|
||||
}
|
||||
|
||||
fn sign_rootfs_tar_image_ref(secret: &str, file_url: &str, digest: &str) -> Result<Vec<u8>> {
|
||||
let mut mac = rootfs_tar_image_ref_hmac(secret)?;
|
||||
mac.update(rootfs_tar_image_ref_message(file_url, digest).as_bytes());
|
||||
Ok(mac.finalize().into_bytes().to_vec())
|
||||
}
|
||||
|
||||
fn verify_rootfs_tar_image_ref_signature(
|
||||
secret: &str,
|
||||
file_url: &str,
|
||||
digest: &str,
|
||||
signature: &[u8],
|
||||
) -> Result<()> {
|
||||
let mut mac = rootfs_tar_image_ref_hmac(secret)?;
|
||||
mac.update(rootfs_tar_image_ref_message(file_url, digest).as_bytes());
|
||||
mac.verify_slice(signature)
|
||||
.map_err(|_| miette::miette!("rootfs artifact signature verification failed"))
|
||||
}
|
||||
|
||||
/// Push a locally-built image into the gateway's containerd runtime.
|
||||
#[allow(clippy::implicit_hasher)]
|
||||
pub async fn push_image_into_gateway(
|
||||
@@ -360,79 +153,6 @@ async fn build_image(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn export_local_image_rootfs_to_path(image_ref: &str, tar_path: &Path) -> Result<()> {
|
||||
let docker = Docker::connect_with_local_defaults()
|
||||
.into_diagnostic()
|
||||
.wrap_err("failed to connect to local Docker daemon")?;
|
||||
let container_name = format!(
|
||||
"openshell-rootfs-export-{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
);
|
||||
let create_options = CreateContainerOptionsBuilder::default()
|
||||
.name(container_name.as_str())
|
||||
.build();
|
||||
let container = docker
|
||||
.create_container(
|
||||
Some(create_options),
|
||||
ContainerCreateBody {
|
||||
image: Some(image_ref.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| {
|
||||
format!("failed to create temporary export container for image {image_ref}")
|
||||
})?;
|
||||
let container_id = container.id;
|
||||
|
||||
let export_result = async {
|
||||
if let Some(parent) = tar_path.parent() {
|
||||
tokio::fs::create_dir_all(parent)
|
||||
.await
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to create {}", parent.display()))?;
|
||||
}
|
||||
let mut file = tokio::fs::File::create(tar_path)
|
||||
.await
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to create {}", tar_path.display()))?;
|
||||
let mut stream = docker.export_container(&container_id);
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to export image {image_ref}"))?;
|
||||
file.write_all(&chunk)
|
||||
.await
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to write {}", tar_path.display()))?;
|
||||
}
|
||||
file.flush()
|
||||
.await
|
||||
.into_diagnostic()
|
||||
.wrap_err_with(|| format!("failed to flush {}", tar_path.display()))
|
||||
}
|
||||
.await;
|
||||
|
||||
let cleanup_result = docker
|
||||
.remove_container(
|
||||
&container_id,
|
||||
Some(RemoveContainerOptionsBuilder::default().force(true).build()),
|
||||
)
|
||||
.await;
|
||||
|
||||
match (export_result, cleanup_result) {
|
||||
(Ok(()), Ok(())) => Ok(()),
|
||||
(Err(err), _) => Err(err),
|
||||
(Ok(()), Err(err)) => Err(err).into_diagnostic().wrap_err_with(|| {
|
||||
format!("failed to remove temporary export container for {image_ref}")
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a tar archive of a directory for use as a Docker build context.
|
||||
///
|
||||
/// Walks `context_dir` recursively, respects a `.dockerignore` file if present,
|
||||
@@ -774,47 +494,4 @@ mod tests {
|
||||
assert!(is_ignored("node_modules", true, &patterns));
|
||||
assert!(is_ignored("node_modules/foo.js", false, &patterns));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encode_and_decode_rootfs_tar_image_ref_round_trip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let tar_path = dir.path().join("rootfs tar.tar");
|
||||
fs::write(&tar_path, "rootfs").unwrap();
|
||||
|
||||
let encoded = encode_rootfs_tar_image_ref(&tar_path).unwrap();
|
||||
let decoded = decode_rootfs_tar_image_ref(&encoded).unwrap();
|
||||
|
||||
assert_eq!(decoded, tar_path.canonicalize().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authenticated_rootfs_tar_image_ref_round_trip() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let tar_path = dir.path().join("rootfs tar.tar");
|
||||
fs::write(&tar_path, "rootfs").unwrap();
|
||||
|
||||
let encoded = encode_authenticated_rootfs_tar_image_ref(&tar_path, "secret").unwrap();
|
||||
let decoded = decode_authenticated_rootfs_tar_image_ref(&encoded, "secret")
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(decoded.path, tar_path.canonicalize().unwrap());
|
||||
assert_eq!(
|
||||
decoded.digest,
|
||||
"sha256:3c47ef972d531d524daa15fa33dd885dd23de6221bbd10a29eb42ecfcf2ef422"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn authenticated_rootfs_tar_image_ref_rejects_wrong_secret() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let tar_path = dir.path().join("rootfs.tar");
|
||||
fs::write(&tar_path, "rootfs").unwrap();
|
||||
|
||||
let encoded = encode_authenticated_rootfs_tar_image_ref(&tar_path, "secret").unwrap();
|
||||
let err = decode_authenticated_rootfs_tar_image_ref(&encoded, "wrong-secret")
|
||||
.expect_err("wrong secret should fail");
|
||||
|
||||
assert!(err.to_string().contains("signature verification failed"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,10 +50,6 @@ pub struct GatewayMetadata {
|
||||
/// Local VM driver state directory for standalone VM gateways.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub vm_driver_state_dir: Option<PathBuf>,
|
||||
|
||||
/// Local secret used to authenticate VM rootfs artifact references.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub vm_rootfs_artifact_secret: Option<String>,
|
||||
}
|
||||
|
||||
impl GatewayMetadata {
|
||||
@@ -145,7 +141,6 @@ pub fn create_gateway_metadata_with_host(
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -474,7 +469,6 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
};
|
||||
let json = serde_json::to_string(&meta).unwrap();
|
||||
let parsed: GatewayMetadata = serde_json::from_str(&json).unwrap();
|
||||
@@ -571,7 +565,6 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
};
|
||||
assert_eq!(meta.gateway_host(), None);
|
||||
}
|
||||
@@ -589,7 +582,6 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
};
|
||||
assert_eq!(meta.gateway_host(), Some("10.0.0.5"));
|
||||
}
|
||||
|
||||
@@ -184,7 +184,6 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
@@ -2846,7 +2846,6 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -898,7 +898,6 @@ fn plaintext_gateway_metadata(
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1101,7 +1100,6 @@ pub async fn gateway_add(
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
};
|
||||
|
||||
store_gateway_metadata(name, &metadata)?;
|
||||
@@ -1132,7 +1130,6 @@ pub async fn gateway_add(
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
};
|
||||
|
||||
store_gateway_metadata(name, &metadata)?;
|
||||
@@ -2663,6 +2660,12 @@ fn image_requests_gpu(image: &str) -> bool {
|
||||
image_name.contains("gpu")
|
||||
}
|
||||
|
||||
const VM_LOCAL_IMAGE_REF_SCHEME: &str = "openshell-vm-local-image";
|
||||
|
||||
fn vm_local_image_ref(image_ref: &str) -> String {
|
||||
format!("{VM_LOCAL_IMAGE_REF_SCHEME}:{image_ref}")
|
||||
}
|
||||
|
||||
fn dockerfile_sources_supported_for_gateway(metadata: Option<&GatewayMetadata>) -> bool {
|
||||
!metadata.is_some_and(|metadata| metadata.is_remote)
|
||||
}
|
||||
@@ -2671,8 +2674,9 @@ fn dockerfile_sources_supported_for_gateway(metadata: Option<&GatewayMetadata>)
|
||||
///
|
||||
/// For local Kubernetes gateways running in Docker, this imports the built image
|
||||
/// into the gateway runtime and returns the Docker tag. For local VM gateways,
|
||||
/// this exports the built image as a rootfs tar artifact and returns an internal
|
||||
/// pseudo-image URI understood by the VM driver.
|
||||
/// this returns an internal local-image URI. The VM driver resolves that URI
|
||||
/// against the local Docker daemon and prepares the VM rootfs on the gateway
|
||||
/// host.
|
||||
async fn build_from_dockerfile(
|
||||
dockerfile: &Path,
|
||||
context: &Path,
|
||||
@@ -2732,34 +2736,18 @@ async fn build_from_dockerfile(
|
||||
return Ok(tag);
|
||||
}
|
||||
|
||||
let rootfs_tar = openshell_bootstrap::build::export_local_image_rootfs(&tag, &mut on_log)
|
||||
.await
|
||||
.wrap_err("failed to export built image as a VM rootfs artifact")?;
|
||||
let artifact_secret = metadata
|
||||
.as_ref()
|
||||
.and_then(|metadata| metadata.vm_rootfs_artifact_secret.as_deref())
|
||||
.filter(|secret| !secret.trim().is_empty())
|
||||
.ok_or_else(|| {
|
||||
miette!(
|
||||
"local Dockerfile sources for VM gateways require authenticated rootfs artifact metadata; restart gateway '{}' with a current `mise run gateway:vm`",
|
||||
gateway_name
|
||||
)
|
||||
})?;
|
||||
let artifact_ref = openshell_bootstrap::build::encode_authenticated_rootfs_tar_image_ref(
|
||||
&rootfs_tar,
|
||||
artifact_secret,
|
||||
)?;
|
||||
let local_image_ref = vm_local_image_ref(&tag);
|
||||
|
||||
eprintln!();
|
||||
eprintln!(
|
||||
"{} VM rootfs artifact {} is ready for gateway '{}'.",
|
||||
"{} Image {} will be resolved by the local VM driver for gateway '{}'.",
|
||||
"✓".green().bold(),
|
||||
rootfs_tar.display().to_string().cyan(),
|
||||
tag.cyan(),
|
||||
gateway_name,
|
||||
);
|
||||
eprintln!();
|
||||
|
||||
Ok(artifact_ref)
|
||||
Ok(local_image_ref)
|
||||
}
|
||||
|
||||
/// Load sandbox policy YAML.
|
||||
@@ -5583,6 +5571,7 @@ mod tests {
|
||||
parse_credential_pairs, plaintext_gateway_is_remote, provisioning_timeout_message,
|
||||
ready_false_condition_message, resolve_gateway_control_target_from, sandbox_should_persist,
|
||||
shell_escape, source_requests_gpu, validate_gateway_name, validate_ssh_host,
|
||||
vm_local_image_ref,
|
||||
};
|
||||
use crate::TEST_ENV_LOCK;
|
||||
use hyper::StatusCode;
|
||||
@@ -5660,7 +5649,6 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5847,7 +5835,6 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
};
|
||||
|
||||
assert!(!dockerfile_sources_supported_for_gateway(Some(&metadata)));
|
||||
@@ -5866,13 +5853,20 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
};
|
||||
|
||||
assert!(dockerfile_sources_supported_for_gateway(Some(&metadata)));
|
||||
assert!(dockerfile_sources_supported_for_gateway(None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn vm_local_image_ref_wraps_docker_image_ref() {
|
||||
assert_eq!(
|
||||
vm_local_image_ref("openshell/sandbox-from:123"),
|
||||
"openshell-vm-local-image:openshell/sandbox-from:123"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_false_condition_message_prefers_reason_and_message() {
|
||||
let status = SandboxStatus {
|
||||
@@ -6089,7 +6083,6 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -6126,7 +6119,6 @@ mod tests {
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
vm_rootfs_artifact_secret: None,
|
||||
};
|
||||
|
||||
assert_eq!(gateway_auth_label(&gateway), "mtls");
|
||||
|
||||
@@ -735,6 +735,10 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() {
|
||||
let _env = test_env(&fake_ssh_dir, &xdg_dir);
|
||||
let tls = test_tls(&server);
|
||||
install_fake_ssh(&fake_ssh_dir);
|
||||
let forward_port = {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
listener.local_addr().unwrap().port()
|
||||
};
|
||||
|
||||
run::sandbox_create(
|
||||
&server.endpoint,
|
||||
@@ -750,7 +754,7 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() {
|
||||
None,
|
||||
&[],
|
||||
None,
|
||||
Some(openshell_core::forward::ForwardSpec::new(8080)),
|
||||
Some(openshell_core::forward::ForwardSpec::new(forward_port)),
|
||||
&["echo".to_string(), "OK".to_string()],
|
||||
Some(false),
|
||||
Some(false),
|
||||
|
||||
@@ -21,8 +21,8 @@ path = "src/main.rs"
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core" }
|
||||
openshell-vfio = { path = "../openshell-vfio" }
|
||||
openshell-bootstrap = { path = "../openshell-bootstrap" }
|
||||
|
||||
bollard = { version = "0.20", features = ["ssh"] }
|
||||
tokio = { workspace = true }
|
||||
tonic = { workspace = true, features = ["transport"] }
|
||||
prost-types = { workspace = true }
|
||||
|
||||
@@ -164,7 +164,9 @@ The VM guest's serial console is appended to `<state-dir>/<sandbox-id>/console.l
|
||||
- Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest)
|
||||
- `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary.
|
||||
- [mise](https://mise.jdx.dev/) task runner
|
||||
- Docker-compatible socket on the CLI host when using `openshell sandbox create --from ./Dockerfile` or `--from ./dir`
|
||||
- Docker-compatible socket on the local CLI/gateway host when using
|
||||
`openshell sandbox create --from ./Dockerfile` or `--from ./dir`; the CLI
|
||||
builds the image and the VM driver exports it via the local Docker daemon
|
||||
- `gh` CLI (used by `mise run vm:setup` to download pre-built runtime artifacts)
|
||||
|
||||
## Relationship to `openshell-vm`
|
||||
|
||||
@@ -8,8 +8,11 @@ use crate::rootfs::{
|
||||
create_rootfs_archive_from_dir, extract_rootfs_archive_to,
|
||||
prepare_sandbox_rootfs_from_image_root, sandbox_guest_init_path,
|
||||
};
|
||||
use bollard::Docker;
|
||||
use bollard::models::ContainerCreateBody;
|
||||
use bollard::query_parameters::{CreateContainerOptionsBuilder, RemoveContainerOptionsBuilder};
|
||||
use flate2::read::GzDecoder;
|
||||
use futures::Stream;
|
||||
use futures::{Stream, StreamExt};
|
||||
use nix::errno::Errno;
|
||||
use nix::sys::signal::{Signal, kill};
|
||||
use nix::unistd::Pid;
|
||||
@@ -17,9 +20,6 @@ use oci_client::client::{Client as OciClient, ClientConfig};
|
||||
use oci_client::manifest::{ImageIndexEntry, OciDescriptor};
|
||||
use oci_client::secrets::RegistryAuth;
|
||||
use oci_client::{Reference, RegistryOperation};
|
||||
use openshell_bootstrap::build::{
|
||||
ROOTFS_TAR_IMAGE_REF_SCHEME, decode_authenticated_rootfs_tar_image_ref,
|
||||
};
|
||||
use openshell_core::proto::compute::v1::{
|
||||
CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse,
|
||||
DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent,
|
||||
@@ -89,8 +89,10 @@ const GUEST_TLS_CERT_PATH: &str = "/opt/openshell/tls/tls.crt";
|
||||
const GUEST_TLS_KEY_PATH: &str = "/opt/openshell/tls/tls.key";
|
||||
const IMAGE_CACHE_ROOT_DIR: &str = "images";
|
||||
const IMAGE_CACHE_ROOTFS_ARCHIVE: &str = "rootfs.tar";
|
||||
const IMAGE_EXPORT_ROOTFS_ARCHIVE: &str = "source-rootfs.tar";
|
||||
const IMAGE_IDENTITY_FILE: &str = "image-identity";
|
||||
const IMAGE_REFERENCE_FILE: &str = "image-reference";
|
||||
const VM_LOCAL_IMAGE_REF_PREFIX: &str = "openshell-vm-local-image:";
|
||||
static IMAGE_CACHE_BUILD_COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -118,7 +120,6 @@ pub struct VmDriverConfig {
|
||||
pub gpu_enabled: bool,
|
||||
pub gpu_mem_mib: u32,
|
||||
pub gpu_vcpus: u8,
|
||||
pub rootfs_artifact_secret: Option<String>,
|
||||
}
|
||||
|
||||
impl Default for VmDriverConfig {
|
||||
@@ -140,7 +141,6 @@ impl Default for VmDriverConfig {
|
||||
gpu_enabled: false,
|
||||
gpu_mem_mib: 8192,
|
||||
gpu_vcpus: 4,
|
||||
rootfs_artifact_secret: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -760,32 +760,9 @@ impl VmDriver {
|
||||
sandbox_id: &str,
|
||||
image_ref: &str,
|
||||
) -> Result<String, Status> {
|
||||
if is_rootfs_tar_image_ref(image_ref) {
|
||||
let secret = self
|
||||
.config
|
||||
.rootfs_artifact_secret
|
||||
.as_deref()
|
||||
.filter(|secret| !secret.trim().is_empty())
|
||||
.ok_or_else(|| {
|
||||
Status::failed_precondition(
|
||||
"vm rootfs tar image references require a gateway-issued artifact secret",
|
||||
)
|
||||
})?;
|
||||
let artifact = decode_authenticated_rootfs_tar_image_ref(image_ref, secret)
|
||||
.map_err(|err| {
|
||||
Status::failed_precondition(format!(
|
||||
"invalid vm rootfs artifact reference: {err}"
|
||||
))
|
||||
})?
|
||||
.ok_or_else(|| {
|
||||
Status::failed_precondition("invalid vm rootfs artifact reference")
|
||||
})?;
|
||||
if let Some(local_image_ref) = parse_vm_local_image_ref(image_ref)? {
|
||||
return self
|
||||
.ensure_cached_rootfs_tar_image_rootfs_archive(
|
||||
image_ref,
|
||||
&artifact.path,
|
||||
&artifact.digest,
|
||||
)
|
||||
.ensure_cached_local_image_rootfs_archive(sandbox_id, image_ref, local_image_ref)
|
||||
.await;
|
||||
}
|
||||
|
||||
@@ -876,58 +853,60 @@ impl VmDriver {
|
||||
Ok(image_identity)
|
||||
}
|
||||
|
||||
async fn ensure_cached_rootfs_tar_image_rootfs_archive(
|
||||
async fn ensure_cached_local_image_rootfs_archive(
|
||||
&self,
|
||||
sandbox_id: &str,
|
||||
image_ref: &str,
|
||||
rootfs_tar_path: &Path,
|
||||
expected_digest: &str,
|
||||
local_image_ref: &str,
|
||||
) -> Result<String, Status> {
|
||||
let rootfs_tar = rootfs_tar_path.to_path_buf();
|
||||
let image_identity = tokio::task::spawn_blocking(move || compute_file_sha256(&rootfs_tar))
|
||||
.await
|
||||
.map_err(|err| {
|
||||
Status::internal(format!("rootfs tar digest computation panicked: {err}"))
|
||||
})?
|
||||
.map_err(|err| {
|
||||
Status::failed_precondition(format!(
|
||||
"failed to fingerprint vm sandbox rootfs artifact '{}': {err}",
|
||||
rootfs_tar_path.display()
|
||||
))
|
||||
})?;
|
||||
if image_identity != expected_digest {
|
||||
return Err(Status::failed_precondition(
|
||||
"vm rootfs artifact digest does not match the authenticated reference",
|
||||
));
|
||||
}
|
||||
let docker = Docker::connect_with_local_defaults().map_err(|err| {
|
||||
Status::failed_precondition(format!(
|
||||
"failed to connect to local Docker daemon for vm local image '{image_ref}': {err}"
|
||||
))
|
||||
})?;
|
||||
let image_identity = local_docker_image_identity(&docker, local_image_ref).await?;
|
||||
let archive_path = image_cache_rootfs_archive(&self.config.state_dir, &image_identity);
|
||||
|
||||
self.publish_platform_event(
|
||||
sandbox_id.to_string(),
|
||||
platform_event(
|
||||
"vm",
|
||||
"Normal",
|
||||
"Pulling",
|
||||
format!("Pulling image \"{local_image_ref}\""),
|
||||
),
|
||||
);
|
||||
|
||||
if tokio::fs::metadata(&archive_path).await.is_ok() {
|
||||
self.publish_pulled_event(sandbox_id, local_image_ref, &archive_path)
|
||||
.await;
|
||||
return Ok(image_identity);
|
||||
}
|
||||
|
||||
let _cache_guard = self.image_cache_lock.lock().await;
|
||||
if tokio::fs::metadata(&archive_path).await.is_ok() {
|
||||
self.publish_pulled_event(sandbox_id, local_image_ref, &archive_path)
|
||||
.await;
|
||||
return Ok(image_identity);
|
||||
}
|
||||
|
||||
self.build_cached_rootfs_tar_image_rootfs_archive(
|
||||
image_ref,
|
||||
rootfs_tar_path,
|
||||
&image_identity,
|
||||
)
|
||||
.await?;
|
||||
self.build_cached_local_image_rootfs_archive(&docker, local_image_ref, &image_identity)
|
||||
.await?;
|
||||
self.publish_pulled_event(sandbox_id, local_image_ref, &archive_path)
|
||||
.await;
|
||||
Ok(image_identity)
|
||||
}
|
||||
|
||||
async fn build_cached_rootfs_tar_image_rootfs_archive(
|
||||
async fn build_cached_local_image_rootfs_archive(
|
||||
&self,
|
||||
docker: &Docker,
|
||||
image_ref: &str,
|
||||
rootfs_tar_path: &Path,
|
||||
image_identity: &str,
|
||||
) -> Result<(), Status> {
|
||||
let cache_dir = image_cache_dir(&self.config.state_dir, image_identity);
|
||||
let archive_path = image_cache_rootfs_archive(&self.config.state_dir, image_identity);
|
||||
let staging_dir = image_cache_staging_dir(&self.config.state_dir, image_identity);
|
||||
let exported_rootfs = staging_dir.join(IMAGE_EXPORT_ROOTFS_ARCHIVE);
|
||||
let prepared_rootfs = staging_dir.join("rootfs");
|
||||
let prepared_archive = staging_dir.join(IMAGE_CACHE_ROOTFS_ARCHIVE);
|
||||
|
||||
@@ -953,24 +932,29 @@ impl VmDriver {
|
||||
Status::internal(format!("create image cache staging dir failed: {err}"))
|
||||
})?;
|
||||
|
||||
if let Err(err) =
|
||||
export_local_image_rootfs_to_path(docker, image_ref, &exported_rootfs).await
|
||||
{
|
||||
let _ = tokio::fs::remove_dir_all(&staging_dir).await;
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let image_ref_owned = image_ref.to_string();
|
||||
let image_identity_owned = image_identity.to_string();
|
||||
let rootfs_tar_path_owned = rootfs_tar_path.to_path_buf();
|
||||
let exported_rootfs_for_build = exported_rootfs.clone();
|
||||
let prepared_rootfs_for_build = prepared_rootfs.clone();
|
||||
let prepared_archive_for_build = prepared_archive.clone();
|
||||
let build_result = tokio::task::spawn_blocking(move || {
|
||||
extract_rootfs_archive_to(&rootfs_tar_path_owned, &prepared_rootfs_for_build)?;
|
||||
prepare_sandbox_rootfs_from_image_root(
|
||||
&prepared_rootfs_for_build,
|
||||
prepare_exported_rootfs_archive(
|
||||
&image_ref_owned,
|
||||
&image_identity_owned,
|
||||
&exported_rootfs_for_build,
|
||||
&prepared_rootfs_for_build,
|
||||
&prepared_archive_for_build,
|
||||
)
|
||||
.map_err(|err| {
|
||||
format!("vm sandbox image '{image_ref_owned}' is not base-compatible: {err}")
|
||||
})?;
|
||||
create_rootfs_archive_from_dir(&prepared_rootfs_for_build, &prepared_archive_for_build)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| Status::internal(format!("rootfs artifact preparation panicked: {err}")))?;
|
||||
.map_err(|err| Status::internal(format!("local image preparation panicked: {err}")))?;
|
||||
|
||||
if let Err(err) = build_result {
|
||||
let _ = tokio::fs::remove_dir_all(&staging_dir).await;
|
||||
@@ -1436,10 +1420,122 @@ fn parse_registry_reference(image_ref: &str) -> Result<Reference, Status> {
|
||||
})
|
||||
}
|
||||
|
||||
fn is_rootfs_tar_image_ref(image_ref: &str) -> bool {
|
||||
image_ref
|
||||
.strip_prefix(ROOTFS_TAR_IMAGE_REF_SCHEME)
|
||||
.is_some_and(|rest| rest.starts_with(':'))
|
||||
#[allow(clippy::result_large_err)]
|
||||
fn parse_vm_local_image_ref(image_ref: &str) -> Result<Option<&str>, Status> {
|
||||
let Some(local_image_ref) = image_ref.strip_prefix(VM_LOCAL_IMAGE_REF_PREFIX) else {
|
||||
return Ok(None);
|
||||
};
|
||||
if local_image_ref.trim().is_empty() {
|
||||
return Err(Status::failed_precondition(
|
||||
"invalid vm local image reference: missing Docker image reference",
|
||||
));
|
||||
}
|
||||
Ok(Some(local_image_ref))
|
||||
}
|
||||
|
||||
async fn local_docker_image_identity(docker: &Docker, image_ref: &str) -> Result<String, Status> {
|
||||
let inspect = docker.inspect_image(image_ref).await.map_err(|err| {
|
||||
Status::failed_precondition(format!(
|
||||
"failed to inspect local Docker image '{image_ref}': {err}"
|
||||
))
|
||||
})?;
|
||||
inspect
|
||||
.id
|
||||
.filter(|id| !id.trim().is_empty())
|
||||
.ok_or_else(|| {
|
||||
Status::failed_precondition(format!(
|
||||
"local Docker image '{image_ref}' inspect response has no image ID"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
async fn export_local_image_rootfs_to_path(
|
||||
docker: &Docker,
|
||||
image_ref: &str,
|
||||
tar_path: &Path,
|
||||
) -> Result<(), Status> {
|
||||
let container_name = format!(
|
||||
"openshell-vm-rootfs-export-{}",
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
);
|
||||
let create_options = CreateContainerOptionsBuilder::default()
|
||||
.name(container_name.as_str())
|
||||
.build();
|
||||
let container = docker
|
||||
.create_container(
|
||||
Some(create_options),
|
||||
ContainerCreateBody {
|
||||
image: Some(image_ref.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
Status::failed_precondition(format!(
|
||||
"failed to create temporary export container for local Docker image '{image_ref}': {err}"
|
||||
))
|
||||
})?;
|
||||
let container_id = container.id;
|
||||
|
||||
let export_result = async {
|
||||
if let Some(parent) = tar_path.parent() {
|
||||
tokio::fs::create_dir_all(parent).await.map_err(|err| {
|
||||
Status::internal(format!(
|
||||
"create export dir {} failed: {err}",
|
||||
parent.display()
|
||||
))
|
||||
})?;
|
||||
}
|
||||
let mut file = tokio::fs::File::create(tar_path).await.map_err(|err| {
|
||||
Status::internal(format!("create {} failed: {err}", tar_path.display()))
|
||||
})?;
|
||||
let mut stream = docker.export_container(&container_id);
|
||||
while let Some(chunk) = stream.next().await {
|
||||
let chunk = chunk.map_err(|err| {
|
||||
Status::failed_precondition(format!(
|
||||
"failed to export local Docker image '{image_ref}': {err}"
|
||||
))
|
||||
})?;
|
||||
file.write_all(&chunk).await.map_err(|err| {
|
||||
Status::internal(format!("write {} failed: {err}", tar_path.display()))
|
||||
})?;
|
||||
}
|
||||
file.flush()
|
||||
.await
|
||||
.map_err(|err| Status::internal(format!("flush {} failed: {err}", tar_path.display())))
|
||||
}
|
||||
.await;
|
||||
|
||||
let cleanup_result = docker
|
||||
.remove_container(
|
||||
&container_id,
|
||||
Some(RemoveContainerOptionsBuilder::default().force(true).build()),
|
||||
)
|
||||
.await;
|
||||
|
||||
match (export_result, cleanup_result) {
|
||||
(Ok(()), Ok(())) => Ok(()),
|
||||
(Err(err), _) => Err(err),
|
||||
(Ok(()), Err(err)) => Err(Status::internal(format!(
|
||||
"failed to remove temporary export container for local Docker image '{image_ref}': {err}"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_exported_rootfs_archive(
|
||||
image_ref: &str,
|
||||
image_identity: &str,
|
||||
exported_rootfs: &Path,
|
||||
prepared_rootfs: &Path,
|
||||
prepared_archive: &Path,
|
||||
) -> Result<(), String> {
|
||||
extract_rootfs_archive_to(exported_rootfs, prepared_rootfs)?;
|
||||
prepare_sandbox_rootfs_from_image_root(prepared_rootfs, image_identity)
|
||||
.map_err(|err| format!("vm sandbox image '{image_ref}' is not base-compatible: {err}"))?;
|
||||
create_rootfs_archive_from_dir(prepared_rootfs, prepared_archive)
|
||||
}
|
||||
|
||||
fn registry_client() -> OciClient {
|
||||
@@ -1700,10 +1796,6 @@ fn verify_descriptor_digest(path: &Path, expected_digest: &str) -> Result<(), St
|
||||
}
|
||||
}
|
||||
|
||||
fn compute_file_sha256(path: &Path) -> Result<String, String> {
|
||||
compute_file_sha256_hex(path).map(|digest| format!("sha256:{digest}"))
|
||||
}
|
||||
|
||||
fn compute_file_sha256_hex(path: &Path) -> Result<String, String> {
|
||||
let mut file = fs::File::open(path).map_err(|err| format!("open {}: {err}", path.display()))?;
|
||||
let mut hasher = Sha256::new();
|
||||
@@ -2585,29 +2677,28 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn ensure_cached_image_rootfs_archive_rejects_unsigned_rootfs_tar_refs() {
|
||||
let base = unique_temp_dir();
|
||||
fs::create_dir_all(&base).unwrap();
|
||||
let artifact = base.join("rootfs.tar");
|
||||
fs::write(&artifact, "not a real rootfs").unwrap();
|
||||
let image_ref = openshell_bootstrap::build::encode_rootfs_tar_image_ref(&artifact).unwrap();
|
||||
let driver = test_driver(VmDriverConfig {
|
||||
state_dir: base.join("driver-state"),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let err = driver
|
||||
.ensure_cached_image_rootfs_archive("sandbox-123", &image_ref)
|
||||
.await
|
||||
.expect_err("unsigned rootfs tar refs must be rejected");
|
||||
|
||||
assert_eq!(err.code(), Code::FailedPrecondition);
|
||||
assert!(
|
||||
err.message().contains("gateway-issued artifact secret"),
|
||||
"unexpected error: {}",
|
||||
err.message()
|
||||
#[test]
|
||||
fn parse_vm_local_image_ref_classifies_prefixed_refs() {
|
||||
assert_eq!(
|
||||
parse_vm_local_image_ref("openshell-vm-local-image:openshell/sandbox-from:123")
|
||||
.unwrap(),
|
||||
Some("openshell/sandbox-from:123")
|
||||
);
|
||||
assert_eq!(parse_vm_local_image_ref("ubuntu:24.04").unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_vm_local_image_ref_rejects_empty_refs() {
|
||||
for image_ref in ["openshell-vm-local-image:", "openshell-vm-local-image: "] {
|
||||
let err =
|
||||
parse_vm_local_image_ref(image_ref).expect_err("empty local image refs must fail");
|
||||
assert_eq!(err.code(), Code::FailedPrecondition);
|
||||
assert!(
|
||||
err.message().contains("missing Docker image reference"),
|
||||
"unexpected error: {}",
|
||||
err.message()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2778,15 +2869,56 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compute_file_sha256_returns_prefixed_digest() {
|
||||
fn prepare_exported_rootfs_archive_rewrites_docker_exported_rootfs() {
|
||||
let base = unique_temp_dir();
|
||||
fs::create_dir_all(&base).unwrap();
|
||||
let file = base.join("rootfs.tar");
|
||||
fs::write(&file, b"openshell").unwrap();
|
||||
let source_rootfs = base.join("source-rootfs");
|
||||
let exported_rootfs = base.join("exported-rootfs.tar");
|
||||
let prepared_rootfs = base.join("prepared-rootfs");
|
||||
let prepared_archive = base.join("prepared-rootfs.tar");
|
||||
let extracted = base.join("extracted");
|
||||
|
||||
for path in [
|
||||
"bin/bash",
|
||||
"bin/mount",
|
||||
"bin/sed",
|
||||
"sbin/ip",
|
||||
"opt/openshell/bin/openshell-sandbox",
|
||||
"usr/local/bin/k3s",
|
||||
] {
|
||||
let path = source_rootfs.join(path);
|
||||
fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
fs::write(path, "").unwrap();
|
||||
}
|
||||
fs::create_dir_all(source_rootfs.join("opt/openshell/manifests")).unwrap();
|
||||
fs::write(source_rootfs.join("opt/openshell/manifests/old.yaml"), "").unwrap();
|
||||
|
||||
create_rootfs_archive_from_dir(&source_rootfs, &exported_rootfs).unwrap();
|
||||
prepare_exported_rootfs_archive(
|
||||
"openshell/sandbox-from:123",
|
||||
"sha256:local-image",
|
||||
&exported_rootfs,
|
||||
&prepared_rootfs,
|
||||
&prepared_archive,
|
||||
)
|
||||
.unwrap();
|
||||
extract_rootfs_archive_to(&prepared_archive, &extracted).unwrap();
|
||||
|
||||
assert!(extracted.join("srv/openshell-vm-sandbox-init.sh").is_file());
|
||||
assert!(
|
||||
extracted
|
||||
.join("opt/openshell/bin/openshell-sandbox")
|
||||
.is_file()
|
||||
);
|
||||
assert!(!extracted.join("usr/local/bin/k3s").exists());
|
||||
assert!(!extracted.join("opt/openshell/manifests").exists());
|
||||
assert_eq!(
|
||||
compute_file_sha256(&file).unwrap(),
|
||||
"sha256:dc5cbc21a452a783ec453e8a8603101dfec5c7d6a19b6c645889bec8b97c2390"
|
||||
fs::read_to_string(extracted.join("opt/openshell/.rootfs-type")).unwrap(),
|
||||
"sandbox\n"
|
||||
);
|
||||
assert!(
|
||||
fs::read_to_string(extracted.join(".openshell-rootfs-variant"))
|
||||
.unwrap()
|
||||
.contains("sha256:local-image")
|
||||
);
|
||||
|
||||
let _ = fs::remove_dir_all(base);
|
||||
@@ -2899,21 +3031,6 @@ mod tests {
|
||||
))
|
||||
}
|
||||
|
||||
fn test_driver(config: VmDriverConfig) -> VmDriver {
|
||||
VmDriver {
|
||||
config,
|
||||
launcher_bin: PathBuf::from("openshell-driver-vm"),
|
||||
registry: Arc::new(Mutex::new(HashMap::new())),
|
||||
image_cache_lock: Arc::new(Mutex::new(())),
|
||||
events: broadcast::channel(WATCH_BUFFER).0,
|
||||
gpu_inventory: None,
|
||||
subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
|
||||
Ipv4Addr::new(10, 0, 128, 0),
|
||||
17,
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn spawn_exited_child() -> Child {
|
||||
Command::new("sh")
|
||||
.arg("-c")
|
||||
|
||||
@@ -105,9 +105,6 @@ struct Args {
|
||||
#[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)]
|
||||
gpu_vcpus: u8,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET", hide = true)]
|
||||
rootfs_artifact_secret: Option<String>,
|
||||
|
||||
#[arg(long, hide = true)]
|
||||
vm_backend: Option<String>,
|
||||
|
||||
@@ -188,7 +185,6 @@ async fn main() -> Result<()> {
|
||||
gpu_enabled: args.gpu,
|
||||
gpu_mem_mib: args.gpu_mem_mib,
|
||||
gpu_vcpus: args.gpu_vcpus,
|
||||
rootfs_artifact_secret: args.rootfs_artifact_secret,
|
||||
})
|
||||
.await
|
||||
.map_err(|err| miette::miette!("{err}"))?;
|
||||
|
||||
@@ -1733,6 +1733,7 @@ fn bootstrap_gateway(rootfs: &Path, gateway_name: &str, gateway_port: u16) -> Re
|
||||
auth_mode: None,
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
};
|
||||
|
||||
let exec_socket = vm_exec_socket_path(rootfs);
|
||||
|
||||
@@ -51,6 +51,10 @@ openshell sandbox create --from my-registry.example.com/my-image:latest
|
||||
|
||||
The CLI resolves community names against the [OpenShell Community](https://github.com/NVIDIA/OpenShell-Community) catalog, pulls the bundled Dockerfile and policy, builds the image locally, and creates the sandbox. For the full catalog and how to contribute your own, refer to [Community Sandboxes](/sandboxes/community-sandboxes).
|
||||
|
||||
Local directories and Dockerfiles require a local gateway because the CLI builds
|
||||
through the local Docker daemon. Use a registry image reference for remote
|
||||
gateways.
|
||||
|
||||
### Label a Sandbox
|
||||
|
||||
Attach labels when you create a sandbox to track ownership, environment, or workflow grouping:
|
||||
|
||||
@@ -85,7 +85,6 @@ register_gateway_metadata() {
|
||||
local endpoint=$2
|
||||
local port=$3
|
||||
local vm_driver_state_dir=$4
|
||||
local rootfs_artifact_secret=$5
|
||||
local config_home gateway_dir
|
||||
|
||||
config_home="${XDG_CONFIG_HOME:-${HOME}/.config}"
|
||||
@@ -100,8 +99,7 @@ register_gateway_metadata() {
|
||||
"is_remote": false,
|
||||
"gateway_port": ${port},
|
||||
"auth_mode": "plaintext",
|
||||
"vm_driver_state_dir": "${vm_driver_state_dir}",
|
||||
"vm_rootfs_artifact_secret": "${rootfs_artifact_secret}"
|
||||
"vm_driver_state_dir": "${vm_driver_state_dir}"
|
||||
}
|
||||
EOF
|
||||
chmod 600 "${gateway_dir}/metadata.json" 2>/dev/null || true
|
||||
@@ -152,10 +150,6 @@ check_supervisor_cross_toolchain() {
|
||||
fi
|
||||
}
|
||||
|
||||
generate_rootfs_artifact_secret() {
|
||||
od -An -N32 -tx1 /dev/urandom | tr -d ' \n'
|
||||
}
|
||||
|
||||
VM_GPU="$(normalize_bool "${OPENSHELL_VM_GPU:-false}")"
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
@@ -219,12 +213,6 @@ VM_DRIVER_STATE_DIR_DEFAULT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}/openshell-v
|
||||
VM_DRIVER_STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${VM_DRIVER_STATE_DIR_DEFAULT}}"
|
||||
|
||||
DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")"
|
||||
ROOTFS_ARTIFACT_SECRET="${OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET:-$(generate_rootfs_artifact_secret)}"
|
||||
if [[ ! "${ROOTFS_ARTIFACT_SECRET}" =~ ^[A-Za-z0-9._~=-]+$ ]]; then
|
||||
echo "ERROR: OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET must contain only URL-safe characters" >&2
|
||||
exit 2
|
||||
fi
|
||||
export OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET="${ROOTFS_ARTIFACT_SECRET}"
|
||||
|
||||
# Build prerequisites: VM runtime artifacts + bundled supervisor.
|
||||
if [ ! -d "${COMPRESSED_DIR}" ] \
|
||||
@@ -264,7 +252,7 @@ mkdir -p "${STATE_DIR}"
|
||||
mkdir -p "${VM_DRIVER_STATE_DIR}"
|
||||
|
||||
GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}"
|
||||
register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}" "${ROOTFS_ARTIFACT_SECRET}"
|
||||
register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}"
|
||||
save_active_gateway "${GATEWAY_NAME}"
|
||||
|
||||
echo "Starting standalone VM gateway..."
|
||||
|
||||
Reference in New Issue
Block a user