trending/OpenShell
1
0
Fork 0
mirror of https://github.com/NVIDIA/OpenShell.git synced 2026-10-29 07:23:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

cleanup

Browse Source
This commit is contained in:
Drew Newberry
2026-04-29 19:14:29 -07:00
parent 616584bba5
commit 7d73d335fb
15 changed files with 437 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Cargo.lock
Generated
+2
View File
@@ -3278,11 +3278,13 @@ dependencies = [
"bollard",
"bytes",
"futures",
"hmac",
"miette",
"openshell-core",
"rcgen",
"serde",
"serde_json",
"sha2 0.10.9",
"tar",
"tempfile",
"tokio",
architecture/sandbox-custom-containers.md
+3 -2
View File
@@ -33,8 +33,9 @@ The community registry prefix defaults to `ghcr.io/nvidia/openshell-community/sa
When `--from` points to a Dockerfile or directory, the CLI:
1. Builds the image locally via the Docker daemon (respecting `.dockerignore`).
2. Pushes it into the cluster's containerd runtime using `docker save` / `ctr import`.
3. Creates the sandbox with the resulting image tag.
2. For a local Kubernetes gateway, pushes it into the cluster's containerd runtime using `docker save` / `ctr import`.
3. For a local VM gateway, exports the built image as a rootfs tar artifact and passes the VM driver a gateway-signed internal artifact reference. The driver rejects unsigned `openshell-rootfs-tar:` references so API clients cannot point it at arbitrary gateway host files.
4. Creates the sandbox with the resulting image tag or VM artifact reference.
## How It Works
crates/openshell-bootstrap/Cargo.toml
+2
View File
@@ -16,10 +16,12 @@ base64 = "0.22"
bollard = { version = "0.20", features = ["ssh"] }
bytes = { workspace = true }
futures = { workspace = true }
hmac = "0.12"
miette = { workspace = true }
rcgen = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sha2 = "0.10"
url = { workspace = true }
tar = "0.4"
tempfile = "3"
crates/openshell-bootstrap/src/build.rs
+178
View File
@@ -9,15 +9,21 @@
//! runtime, while the VM backend can export the built image as a rootfs tar.
use std::collections::HashMap;
use std::fs::File;
use std::io::Read;
use std::path::{Path, PathBuf};
use base64::Engine as _;
use base64::engine::general_purpose::URL_SAFE_NO_PAD;
use bollard::Docker;
use bollard::models::ContainerCreateBody;
use bollard::query_parameters::{
BuildImageOptionsBuilder, CreateContainerOptionsBuilder, RemoveContainerOptionsBuilder,
};
use futures::StreamExt;
use hmac::{Hmac, Mac};
use miette::{IntoDiagnostic, Result, WrapErr};
use sha2::{Digest, Sha256};
use tokio::io::AsyncWriteExt;
use url::{Position, Url};
@@ -26,6 +32,15 @@ use crate::push::push_local_images;
/// Pseudo-image URI scheme used to hand a local rootfs tar artifact to the VM driver.
pub const ROOTFS_TAR_IMAGE_REF_SCHEME: &str = "openshell-rootfs-tar";
const ROOTFS_TAR_IMAGE_REF_VERSION: &str = "v1";
type HmacSha256 = Hmac<Sha256>;
/// Authenticated VM rootfs tar artifact decoded from an internal image reference.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct RootfsTarImageRef {
pub path: PathBuf,
pub digest: String,
}
/// Build a container image from a Dockerfile using the local Docker daemon.
///
@@ -65,6 +80,25 @@ pub fn encode_rootfs_tar_image_ref(path: &Path) -> Result<String> {
))
}
/// Encode a local rootfs tar path as an authenticated VM-driver artifact reference.
pub fn encode_authenticated_rootfs_tar_image_ref(path: &Path, secret: &str) -> Result<String> {
let canonical = path
.canonicalize()
.into_diagnostic()
.wrap_err_with(|| format!("failed to resolve rootfs tar path {}", path.display()))?;
let digest = compute_file_sha256_hex(&canonical)?;
let file_url = Url::from_file_path(&canonical).map_err(|()| {
miette::miette!("failed to encode rootfs tar path {}", canonical.display())
})?;
let file_url = file_url.to_string();
let signature = sign_rootfs_tar_image_ref(secret, &file_url, &digest)?;
Ok(format!(
"{ROOTFS_TAR_IMAGE_REF_SCHEME}:{ROOTFS_TAR_IMAGE_REF_VERSION}:{}:{digest}:{}",
URL_SAFE_NO_PAD.encode(file_url.as_bytes()),
URL_SAFE_NO_PAD.encode(signature),
))
}
/// Decode a VM-driver rootfs tar image reference back to a local filesystem path.
pub fn decode_rootfs_tar_image_ref(image_ref: &str) -> Option<PathBuf> {
let remainder = image_ref.strip_prefix(&format!("{ROOTFS_TAR_IMAGE_REF_SCHEME}:"))?;
@@ -72,6 +106,62 @@ pub fn decode_rootfs_tar_image_ref(image_ref: &str) -> Option<PathBuf> {
Url::parse(&file_url).ok()?.to_file_path().ok()
}
/// Decode and verify an authenticated VM-driver rootfs tar image reference.
pub fn decode_authenticated_rootfs_tar_image_ref(
image_ref: &str,
secret: &str,
) -> Result<Option<RootfsTarImageRef>> {
let Some(remainder) = image_ref.strip_prefix(&format!(
"{ROOTFS_TAR_IMAGE_REF_SCHEME}:{ROOTFS_TAR_IMAGE_REF_VERSION}:"
)) else {
return Ok(None);
};
let mut parts = remainder.split(':');
let encoded_url = parts
.next()
.filter(|value| !value.is_empty())
.ok_or_else(|| miette::miette!("missing rootfs artifact path"))?;
let digest = parts
.next()
.filter(|value| !value.is_empty())
.ok_or_else(|| miette::miette!("missing rootfs artifact digest"))?;
let encoded_signature = parts
.next()
.filter(|value| !value.is_empty())
.ok_or_else(|| miette::miette!("missing rootfs artifact signature"))?;
if parts.next().is_some() {
return Err(miette::miette!("malformed rootfs artifact reference"));
}
validate_sha256_hex(digest)?;
let file_url_bytes = URL_SAFE_NO_PAD
.decode(encoded_url)
.map_err(|err| miette::miette!("invalid rootfs artifact path encoding: {err}"))?;
let file_url = String::from_utf8(file_url_bytes)
.map_err(|err| miette::miette!("invalid rootfs artifact path encoding: {err}"))?;
let signature = URL_SAFE_NO_PAD
.decode(encoded_signature)
.map_err(|err| miette::miette!("invalid rootfs artifact signature encoding: {err}"))?;
verify_rootfs_tar_image_ref_signature(secret, &file_url, digest, &signature)?;
let url = Url::parse(&file_url)
.into_diagnostic()
.wrap_err("invalid rootfs artifact URL")?;
if url.scheme() != "file" {
return Err(miette::miette!("rootfs artifact URL must use file scheme"));
}
let path = url
.to_file_path()
.map_err(|()| miette::miette!("rootfs artifact URL is not a local file path"))?;
Ok(Some(RootfsTarImageRef {
path,
digest: format!("sha256:{digest}"),
}))
}
/// Export a locally-built Docker image as a persistent rootfs tar artifact for the VM driver.
pub async fn export_local_image_rootfs(
image_ref: &str,
@@ -103,6 +193,63 @@ pub async fn export_local_image_rootfs(
Ok(output_path)
}
fn compute_file_sha256_hex(path: &Path) -> Result<String> {
let mut file = File::open(path)
.into_diagnostic()
.wrap_err_with(|| format!("failed to open {}", path.display()))?;
let mut hasher = Sha256::new();
let mut buf = [0u8; 64 * 1024];
loop {
let read = file
.read(&mut buf)
.into_diagnostic()
.wrap_err_with(|| format!("failed to read {}", path.display()))?;
if read == 0 {
break;
}
hasher.update(&buf[..read]);
}
Ok(format!("{:x}", hasher.finalize()))
}
fn validate_sha256_hex(value: &str) -> Result<()> {
if value.len() == 64 && value.bytes().all(|byte| byte.is_ascii_hexdigit()) {
Ok(())
} else {
Err(miette::miette!("rootfs artifact digest must be sha256 hex"))
}
}
fn rootfs_tar_image_ref_message(file_url: &str, digest: &str) -> String {
format!("{ROOTFS_TAR_IMAGE_REF_VERSION}\n{file_url}\nsha256:{digest}")
}
fn rootfs_tar_image_ref_hmac(secret: &str) -> Result<HmacSha256> {
if secret.trim().is_empty() {
return Err(miette::miette!("rootfs artifact secret is empty"));
}
HmacSha256::new_from_slice(secret.as_bytes())
.map_err(|err| miette::miette!("invalid rootfs artifact secret: {err}"))
}
fn sign_rootfs_tar_image_ref(secret: &str, file_url: &str, digest: &str) -> Result<Vec<u8>> {
let mut mac = rootfs_tar_image_ref_hmac(secret)?;
mac.update(rootfs_tar_image_ref_message(file_url, digest).as_bytes());
Ok(mac.finalize().into_bytes().to_vec())
}
fn verify_rootfs_tar_image_ref_signature(
secret: &str,
file_url: &str,
digest: &str,
signature: &[u8],
) -> Result<()> {
let mut mac = rootfs_tar_image_ref_hmac(secret)?;
mac.update(rootfs_tar_image_ref_message(file_url, digest).as_bytes());
mac.verify_slice(signature)
.map_err(|_| miette::miette!("rootfs artifact signature verification failed"))
}
/// Push a locally-built image into the gateway's containerd runtime.
#[allow(clippy::implicit_hasher)]
pub async fn push_image_into_gateway(
@@ -639,4 +786,35 @@ mod tests {
assert_eq!(decoded, tar_path.canonicalize().unwrap());
}
#[test]
fn authenticated_rootfs_tar_image_ref_round_trip() {
let dir = tempfile::tempdir().unwrap();
let tar_path = dir.path().join("rootfs tar.tar");
fs::write(&tar_path, "rootfs").unwrap();
let encoded = encode_authenticated_rootfs_tar_image_ref(&tar_path, "secret").unwrap();
let decoded = decode_authenticated_rootfs_tar_image_ref(&encoded, "secret")
.unwrap()
.unwrap();
assert_eq!(decoded.path, tar_path.canonicalize().unwrap());
assert_eq!(
decoded.digest,
"sha256:3c47ef972d531d524daa15fa33dd885dd23de6221bbd10a29eb42ecfcf2ef422"
);
}
#[test]
fn authenticated_rootfs_tar_image_ref_rejects_wrong_secret() {
let dir = tempfile::tempdir().unwrap();
let tar_path = dir.path().join("rootfs.tar");
fs::write(&tar_path, "rootfs").unwrap();
let encoded = encode_authenticated_rootfs_tar_image_ref(&tar_path, "secret").unwrap();
let err = decode_authenticated_rootfs_tar_image_ref(&encoded, "wrong-secret")
.expect_err("wrong secret should fail");
assert!(err.to_string().contains("signature verification failed"));
}
}
crates/openshell-bootstrap/src/metadata.rs
+16
View File
@@ -46,6 +46,14 @@ pub struct GatewayMetadata {
alias = "cf_auth_url"
)]
pub edge_auth_url: Option<String>,
/// Local VM driver state directory for standalone VM gateways.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub vm_driver_state_dir: Option<PathBuf>,
/// Local secret used to authenticate VM rootfs artifact references.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub vm_rootfs_artifact_secret: Option<String>,
}
impl GatewayMetadata {
@@ -136,6 +144,8 @@ pub fn create_gateway_metadata_with_host(
auth_mode: disable_tls.then(|| "plaintext".to_string()),
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
}
}
@@ -463,6 +473,8 @@ mod tests {
auth_mode: None,
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
};
let json = serde_json::to_string(&meta).unwrap();
let parsed: GatewayMetadata = serde_json::from_str(&json).unwrap();
@@ -558,6 +570,8 @@ mod tests {
auth_mode: None,
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
};
assert_eq!(meta.gateway_host(), None);
}
@@ -574,6 +588,8 @@ mod tests {
auth_mode: None,
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
};
assert_eq!(meta.gateway_host(), Some("10.0.0.5"));
}
crates/openshell-cli/src/completers.rs
+2
View File
@@ -183,6 +183,8 @@ mod tests {
auth_mode: Some("cloudflare_jwt".to_string()),
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
},
)
.unwrap();
crates/openshell-cli/src/main.rs
+2
View File
@@ -2845,6 +2845,8 @@ mod tests {
auth_mode: Some("cloudflare_jwt".to_string()),
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
}
}
crates/openshell-cli/src/run.rs
+30 -66
View File
@@ -194,46 +194,6 @@ enum KubeEventReason {
Started,
}
/// Decide whether a server/driver log line should be surfaced during
/// `sandbox create` provisioning.
///
/// Defaults to a curated allowlist so the spinner stays focused on
/// meaningful progress. Set `OPENSHELL_PROVISION_VERBOSE=1` to surface
/// every log line (helpful when debugging stuck provisioning, e.g. on
/// the experimental VM gateway).
fn should_show_provisioning_log(line: &openshell_core::proto::SandboxLogLine) -> bool {
// Allowlisted substrings for info-level progress lines emitted by
// the server compute layer and bundled drivers (VM / Docker). Keep
// this short — anything not matching is suppressed by default.
const ALLOWLIST: &[&str] = &[
"Sandbox phase changed",
"Pulling image",
"Pulled image",
"Extracting",
"Preparing rootfs",
"Booting VM",
"Starting VM",
"Starting sandbox",
"Sandbox ready",
"Supervisor connected",
];
if std::env::var("OPENSHELL_PROVISION_VERBOSE")
.map(|v| !v.is_empty() && v != "0" && !v.eq_ignore_ascii_case("false"))
.unwrap_or(false)
{
return true;
}
// Always surface warnings and errors during provisioning.
let level = line.level.to_ascii_lowercase();
if matches!(level.as_str(), "warn" | "warning" | "error") {
return true;
}
ALLOWLIST.iter().any(|needle| line.message.contains(needle))
}
/// Map a Kubernetes event reason string to an enum.
fn parse_kube_event_reason(reason: &str) -> Option<KubeEventReason> {
match reason {
@@ -937,6 +897,8 @@ fn plaintext_gateway_metadata(
auth_mode: Some("plaintext".to_string()),
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
}
}
@@ -1138,6 +1100,8 @@ pub async fn gateway_add(
auth_mode: Some("mtls".to_string()),
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
};
store_gateway_metadata(name, &metadata)?;
@@ -1167,6 +1131,8 @@ pub async fn gateway_add(
auth_mode: Some("cloudflare_jwt".to_string()),
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
};
store_gateway_metadata(name, &metadata)?;
@@ -2342,31 +2308,6 @@ pub async fn sandbox_create(
if !saw_gateway_ready && line.message.contains("listening") {
saw_gateway_ready = true;
}
// Surface log lines as progress so users aren't staring at a
// silent spinner while non-Kubernetes drivers (VM, Docker) do
// their work. Drivers/server tracing with a `sandbox_id`
// field flows through here as Log payloads.
//
// The default filter keeps output focused on user-relevant
// progress: warn/error always, plus a curated allowlist of
// info messages. Set OPENSHELL_PROVISION_VERBOSE=1 to see
// every log line during provisioning.
if !line.message.is_empty() && should_show_provisioning_log(&line) {
if let Some(d) = display.as_mut() {
// Interactive: tuck the message under the spinner
// as detail so the checklist stays clean.
d.set_active_detail(&line.message);
} else {
let ts = format_timestamp(provision_start.elapsed());
let level = if line.level.is_empty() {
"INFO".to_string()
} else {
line.level.to_uppercase()
};
eprintln!(" {} {} {}", ts.dimmed(), level.dimmed(), line.message,);
}
}
}
Some(openshell_core::proto::sandbox_stream_event::Payload::Event(ev)) => {
// Map Kubernetes events to provisioning steps.
@@ -2794,7 +2735,20 @@ async fn build_from_dockerfile(
let rootfs_tar = openshell_bootstrap::build::export_local_image_rootfs(&tag, &mut on_log)
.await
.wrap_err("failed to export built image as a VM rootfs artifact")?;
let artifact_ref = openshell_bootstrap::build::encode_rootfs_tar_image_ref(&rootfs_tar)?;
let artifact_secret = metadata
.as_ref()
.and_then(|metadata| metadata.vm_rootfs_artifact_secret.as_deref())
.filter(|secret| !secret.trim().is_empty())
.ok_or_else(|| {
miette!(
"local Dockerfile sources for VM gateways require authenticated rootfs artifact metadata; restart gateway '{}' with a current `mise run gateway:vm`",
gateway_name
)
})?;
let artifact_ref = openshell_bootstrap::build::encode_authenticated_rootfs_tar_image_ref(
&rootfs_tar,
artifact_secret,
)?;
eprintln!();
eprintln!(
@@ -5705,6 +5659,8 @@ mod tests {
auth_mode: Some("cloudflare_jwt".to_string()),
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
}
}
@@ -5890,6 +5846,8 @@ mod tests {
auth_mode: None,
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
};
assert!(!dockerfile_sources_supported_for_gateway(Some(&metadata)));
@@ -5907,6 +5865,8 @@ mod tests {
auth_mode: None,
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
};
assert!(dockerfile_sources_supported_for_gateway(Some(&metadata)));
@@ -6128,6 +6088,8 @@ mod tests {
auth_mode: None,
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
},
];
@@ -6163,6 +6125,8 @@ mod tests {
auth_mode: None,
edge_team_domain: None,
edge_auth_url: None,
vm_driver_state_dir: None,
vm_rootfs_artifact_secret: None,
};
assert_eq!(gateway_auth_label(&gateway), "mtls");
crates/openshell-driver-vm/src/driver.rs
+96 -5
View File
@@ -17,7 +17,9 @@ use oci_client::client::{Client as OciClient, ClientConfig};
use oci_client::manifest::{ImageIndexEntry, OciDescriptor};
use oci_client::secrets::RegistryAuth;
use oci_client::{Reference, RegistryOperation};
use openshell_bootstrap::build::decode_rootfs_tar_image_ref;
use openshell_bootstrap::build::{
ROOTFS_TAR_IMAGE_REF_SCHEME, decode_authenticated_rootfs_tar_image_ref,
};
use openshell_core::proto::compute::v1::{
CreateSandboxRequest, CreateSandboxResponse, DeleteSandboxRequest, DeleteSandboxResponse,
DriverCondition as SandboxCondition, DriverPlatformEvent as PlatformEvent,
@@ -116,6 +118,7 @@ pub struct VmDriverConfig {
pub gpu_enabled: bool,
pub gpu_mem_mib: u32,
pub gpu_vcpus: u8,
pub rootfs_artifact_secret: Option<String>,
}
impl Default for VmDriverConfig {
@@ -137,6 +140,7 @@ impl Default for VmDriverConfig {
gpu_enabled: false,
gpu_mem_mib: 8192,
gpu_vcpus: 4,
rootfs_artifact_secret: None,
}
}
}
@@ -756,9 +760,32 @@ impl VmDriver {
sandbox_id: &str,
image_ref: &str,
) -> Result<String, Status> {
if let Some(rootfs_tar_path) = decode_rootfs_tar_image_ref(image_ref) {
if is_rootfs_tar_image_ref(image_ref) {
let secret = self
.config
.rootfs_artifact_secret
.as_deref()
.filter(|secret| !secret.trim().is_empty())
.ok_or_else(|| {
Status::failed_precondition(
"vm rootfs tar image references require a gateway-issued artifact secret",
)
})?;
let artifact = decode_authenticated_rootfs_tar_image_ref(image_ref, secret)
.map_err(|err| {
Status::failed_precondition(format!(
"invalid vm rootfs artifact reference: {err}"
))
})?
.ok_or_else(|| {
Status::failed_precondition("invalid vm rootfs artifact reference")
})?;
return self
.ensure_cached_rootfs_tar_image_rootfs_archive(image_ref, &rootfs_tar_path)
.ensure_cached_rootfs_tar_image_rootfs_archive(
image_ref,
&artifact.path,
&artifact.digest,
)
.await;
}
@@ -853,6 +880,7 @@ impl VmDriver {
&self,
image_ref: &str,
rootfs_tar_path: &Path,
expected_digest: &str,
) -> Result<String, Status> {
let rootfs_tar = rootfs_tar_path.to_path_buf();
let image_identity = tokio::task::spawn_blocking(move || compute_file_sha256(&rootfs_tar))
@@ -866,6 +894,11 @@ impl VmDriver {
rootfs_tar_path.display()
))
})?;
if image_identity != expected_digest {
return Err(Status::failed_precondition(
"vm rootfs artifact digest does not match the authenticated reference",
));
}
let archive_path = image_cache_rootfs_archive(&self.config.state_dir, &image_identity);
if tokio::fs::metadata(&archive_path).await.is_ok() {
@@ -1403,6 +1436,12 @@ fn parse_registry_reference(image_ref: &str) -> Result<Reference, Status> {
})
}
fn is_rootfs_tar_image_ref(image_ref: &str) -> bool {
image_ref
.strip_prefix(ROOTFS_TAR_IMAGE_REF_SCHEME)
.is_some_and(|rest| rest.starts_with(':'))
}
fn registry_client() -> OciClient {
OciClient::new(ClientConfig {
platform_resolver: Some(Box::new(linux_platform_resolver)),
@@ -1470,8 +1509,12 @@ fn env_non_empty(key: &str) -> Option<String> {
}
fn image_reference_registry_host(image_ref: &str) -> &str {
let first = image_ref.split('/').next().unwrap_or(image_ref);
if first.contains('.') || first.contains(':') || first.eq_ignore_ascii_case("localhost") {
let mut parts = image_ref.splitn(2, '/');
let first = parts.next().unwrap_or(image_ref);
let has_path = parts.next().is_some();
if has_path
&& (first.contains('.') || first.contains(':') || first.eq_ignore_ascii_case("localhost"))
{
first
} else {
"docker.io"
@@ -2524,16 +2567,49 @@ mod tests {
#[test]
fn image_reference_registry_host_defaults_to_docker_hub() {
assert_eq!(image_reference_registry_host("ubuntu:24.04"), "docker.io");
assert_eq!(
image_reference_registry_host("library/ubuntu:24.04"),
"docker.io"
);
assert_eq!(
image_reference_registry_host("ghcr.io/nvidia/openshell/base:latest"),
"ghcr.io"
);
assert_eq!(
image_reference_registry_host("localhost/example:dev"),
"localhost"
);
assert_eq!(
image_reference_registry_host("localhost:5000/example/sandbox:dev"),
"localhost:5000"
);
}
#[tokio::test]
async fn ensure_cached_image_rootfs_archive_rejects_unsigned_rootfs_tar_refs() {
let base = unique_temp_dir();
fs::create_dir_all(&base).unwrap();
let artifact = base.join("rootfs.tar");
fs::write(&artifact, "not a real rootfs").unwrap();
let image_ref = openshell_bootstrap::build::encode_rootfs_tar_image_ref(&artifact).unwrap();
let driver = test_driver(VmDriverConfig {
state_dir: base.join("driver-state"),
..Default::default()
});
let err = driver
.ensure_cached_image_rootfs_archive("sandbox-123", &image_ref)
.await
.expect_err("unsigned rootfs tar refs must be rejected");
assert_eq!(err.code(), Code::FailedPrecondition);
assert!(
err.message().contains("gateway-issued artifact secret"),
"unexpected error: {}",
err.message()
);
}
#[test]
fn apply_layer_dir_to_rootfs_honors_whiteouts() {
let base = unique_temp_dir();
@@ -2823,6 +2899,21 @@ mod tests {
))
}
fn test_driver(config: VmDriverConfig) -> VmDriver {
VmDriver {
config,
launcher_bin: PathBuf::from("openshell-driver-vm"),
registry: Arc::new(Mutex::new(HashMap::new())),
image_cache_lock: Arc::new(Mutex::new(())),
events: broadcast::channel(WATCH_BUFFER).0,
gpu_inventory: None,
subnet_allocator: Arc::new(std::sync::Mutex::new(SubnetAllocator::new(
Ipv4Addr::new(10, 0, 128, 0),
17,
))),
}
}
fn spawn_exited_child() -> Child {
Command::new("sh")
.arg("-c")
crates/openshell-driver-vm/src/main.rs
+4
View File
@@ -105,6 +105,9 @@ struct Args {
#[arg(long, env = "OPENSHELL_VM_GPU_VCPUS", default_value_t = 4)]
gpu_vcpus: u8,
#[arg(long, env = "OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET", hide = true)]
rootfs_artifact_secret: Option<String>,
#[arg(long, hide = true)]
vm_backend: Option<String>,
@@ -185,6 +188,7 @@ async fn main() -> Result<()> {
gpu_enabled: args.gpu,
gpu_mem_mib: args.gpu_mem_mib,
gpu_vcpus: args.gpu_vcpus,
rootfs_artifact_secret: args.rootfs_artifact_secret,
})
.await
.map_err(|err| miette::miette!("{err}"))?;
crates/openshell-server/src/lib.rs
+34 -43
View File
@@ -148,10 +148,14 @@ pub async fn run_server(
if database_url.is_empty() {
return Err(Error::config("database_url is required"));
}
// The ssh_handshake_secret check is driver-scoped: the Kubernetes and
// Podman drivers inject the secret into sandbox env, while the VM and
// Docker drivers do not consume it. The check is enforced per-driver
// in `build_compute_runtime`.
let driver = configured_compute_driver(&config)?;
if config.ssh_handshake_secret.is_empty()
&& !matches!(driver, ComputeDriverKind::Docker | ComputeDriverKind::Vm)
{
return Err(Error::config(
"ssh_handshake_secret is required. Set --ssh-handshake-secret or OPENSHELL_SSH_HANDSHAKE_SECRET",
));
}
let store = Arc::new(Store::connect(database_url).await?);
@@ -366,35 +370,32 @@ async fn build_compute_runtime(
info!(driver = %driver, "Using compute driver");
match driver {
ComputeDriverKind::Kubernetes => {
require_ssh_handshake_secret(config)?;
ComputeRuntime::new_kubernetes(
KubernetesComputeConfig {
namespace: config.sandbox_namespace.clone(),
default_image: config.sandbox_image.clone(),
image_pull_policy: config.sandbox_image_pull_policy.clone(),
grpc_endpoint: config.grpc_endpoint.clone(),
// Filesystem path to the supervisor's Unix-socket SSH daemon.
// The path lives in a root-only directory so only the
// supervisor can connect; the gateway reaches it through
// the RelayStream bridge, not directly. Override via
// `sandbox_ssh_socket_path` in the config for deployments
// where multiple supervisors share a filesystem.
ssh_socket_path: config.sandbox_ssh_socket_path.clone(),
ssh_handshake_secret: config.ssh_handshake_secret.clone(),
ssh_handshake_skew_secs: config.ssh_handshake_skew_secs,
client_tls_secret_name: config.client_tls_secret_name.clone(),
host_gateway_ip: config.host_gateway_ip.clone(),
},
store,
sandbox_index,
sandbox_watch_bus,
tracing_log_bus,
supervisor_sessions.clone(),
)
.await
.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))
}
ComputeDriverKind::Kubernetes => ComputeRuntime::new_kubernetes(
KubernetesComputeConfig {
namespace: config.sandbox_namespace.clone(),
default_image: config.sandbox_image.clone(),
image_pull_policy: config.sandbox_image_pull_policy.clone(),
grpc_endpoint: config.grpc_endpoint.clone(),
// Filesystem path to the supervisor's Unix-socket SSH daemon.
// The path lives in a root-only directory so only the
// supervisor can connect; the gateway reaches it through the
// RelayStream bridge, not directly. Override via
// `sandbox_ssh_socket_path` in the config for deployments
// where multiple supervisors share a filesystem.
ssh_socket_path: config.sandbox_ssh_socket_path.clone(),
ssh_handshake_secret: config.ssh_handshake_secret.clone(),
ssh_handshake_skew_secs: config.ssh_handshake_skew_secs,
client_tls_secret_name: config.client_tls_secret_name.clone(),
host_gateway_ip: config.host_gateway_ip.clone(),
},
store,
sandbox_index,
sandbox_watch_bus,
tracing_log_bus,
supervisor_sessions.clone(),
)
.await
.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))),
ComputeDriverKind::Docker => ComputeRuntime::new_docker(
config.clone(),
docker_config.clone(),
@@ -421,7 +422,6 @@ async fn build_compute_runtime(
.map_err(|e| Error::execution(format!("failed to create compute runtime: {e}")))
}
ComputeDriverKind::Podman => {
require_ssh_handshake_secret(config)?;
let socket_path = std::env::var("OPENSHELL_PODMAN_SOCKET")
.ok()
.filter(|s| !s.is_empty())
@@ -473,15 +473,6 @@ async fn build_compute_runtime(
}
}
fn require_ssh_handshake_secret(config: &Config) -> Result<()> {
if config.ssh_handshake_secret.is_empty() {
return Err(Error::config(
"ssh_handshake_secret is required for this driver. Set --ssh-handshake-secret or OPENSHELL_SSH_HANDSHAKE_SECRET",
));
}
Ok(())
}
fn configured_compute_driver(config: &Config) -> Result<ComputeDriverKind> {
match config.compute_drivers.as_slice() {
[] => Err(Error::config(
tasks/gateway.toml
-20
View File
@@ -2,26 +2,6 @@
# SPDX-License-Identifier: Apache-2.0
# Standalone gateway tasks
#
# Canonical entry point:
# mise run gateway:start <driver> # driver = docker | vm
#
# Backward-compatible aliases:
# mise run gateway:docker
# mise run gateway:vm
#
# All three commands also write ~/.config/openshell/active_gateway so the
# `openshell` CLI automatically targets the gateway you just started — no
# need to follow up with `openshell gateway select`. Inside this repo you
# can override the active gateway per-developer by setting OPENSHELL_GATEWAY
# in `.env` (mise loads it automatically).
["gateway:start"]
description = "Run a standalone gateway with the given compute driver (docker | vm)"
usage = """
arg "[driver]" help="Compute driver to start (docker | vm). Defaults to $OPENSHELL_GATEWAY_DRIVER or 'docker'."
"""
run = "bash tasks/scripts/gateway-start.sh"
["gateway:docker"]
description = "Run a standalone gateway with the bundled Docker compute driver"
tasks/scripts/gateway-docker.sh
+6 -25
View File
@@ -6,10 +6,6 @@
# Start a standalone openshell-gateway backed by the Docker compute driver for
# local manual testing.
#
# Invocation:
# mise run gateway:start docker # canonical
# mise run gateway:docker # alias
#
# Defaults:
# - Plaintext HTTP on 127.0.0.1:18080
# - Dedicated sandbox namespace "docker-dev"
@@ -21,11 +17,9 @@
# OPENSHELL_SANDBOX_NAMESPACE=my-ns mise run gateway:docker
# OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:docker
#
# This script also writes ~/.config/openshell/active_gateway so the
# `openshell` CLI automatically targets this gateway in subsequent shells.
# No need to run `openshell gateway select`. Inside this repo you can
# override per-developer with OPENSHELL_GATEWAY in `.env` (mise loads it).
# An explicit `--gateway` / `--gateway-endpoint` flag still wins.
# After the gateway is running, point the CLI at it with either:
# openshell --gateway docker-dev <command>
# openshell gateway use docker-dev # then plain `openshell <command>`
set -euo pipefail
@@ -92,18 +86,6 @@ register_gateway_metadata() {
EOF
}
# Mirror what `openshell gateway select <name>` does: write the gateway name
# to $XDG_CONFIG_HOME/openshell/active_gateway. The CLI picks it up as the
# default target when neither --gateway nor OPENSHELL_GATEWAY is set.
save_active_gateway() {
local name=$1
local config_home active_gateway_path
config_home="${XDG_CONFIG_HOME:-${HOME}/.config}"
active_gateway_path="${config_home}/openshell/active_gateway"
mkdir -p "$(dirname "${active_gateway_path}")"
printf '%s' "${name}" >"${active_gateway_path}"
}
if [[ ! "${GATEWAY_NAME}" =~ ^[A-Za-z0-9._-]+$ ]]; then
echo "ERROR: OPENSHELL_DOCKER_GATEWAY_NAME must contain only letters, numbers, dots, underscores, or dashes" >&2
exit 2
@@ -179,7 +161,6 @@ mkdir -p "${STATE_DIR}"
GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}"
register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}"
save_active_gateway "${GATEWAY_NAME}"
echo "Starting standalone Docker gateway..."
echo " gateway: ${GATEWAY_NAME}"
@@ -187,9 +168,9 @@ echo " endpoint: ${GATEWAY_ENDPOINT}"
echo " namespace: ${SANDBOX_NAMESPACE}"
echo " state dir: ${STATE_DIR}"
echo
echo "Active gateway set to '${GATEWAY_NAME}'. The CLI now targets this gateway"
echo "by default — just run \`openshell <command>\`. Override with --gateway"
echo "or by setting OPENSHELL_GATEWAY (e.g. in .env)."
echo "Point the CLI at this gateway with one of:"
echo " openshell --gateway ${GATEWAY_NAME} status"
echo " openshell gateway select ${GATEWAY_NAME}"
echo
exec "${GATEWAY_BIN}" \
tasks/scripts/gateway-start.sh
-34
View File
@@ -1,34 +0,0 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Dispatcher for `mise run gateway:start <driver>`.
#
# Usage:
# mise run gateway:start # uses $OPENSHELL_GATEWAY_DRIVER or 'docker'
# mise run gateway:start docker
# mise run gateway:start vm
#
# This is a thin shim that forwards to the driver-specific script. The
# `gateway:docker` / `gateway:vm` mise aliases call those scripts directly.
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
DRIVER="${1:-${OPENSHELL_GATEWAY_DRIVER:-docker}}"
case "${DRIVER}" in
docker)
exec bash "${ROOT}/tasks/scripts/gateway-docker.sh"
;;
vm)
exec bash "${ROOT}/tasks/scripts/gateway-vm.sh"
;;
*)
echo "ERROR: unknown gateway driver '${DRIVER}' (expected 'docker' or 'vm')" >&2
echo "Usage: mise run gateway:start <docker|vm>" >&2
exit 2
;;
esac
tasks/scripts/gateway-vm.sh
+62 -4
View File
@@ -7,8 +7,7 @@
# (openshell-driver-vm) for local manual testing.
#
# Invocation:
# mise run gateway:start vm # canonical
# mise run gateway:vm # alias
# mise run gateway:vm
#
# Defaults:
# - Plaintext HTTP on 127.0.0.1:18081
@@ -23,6 +22,7 @@
# OPENSHELL_VM_GATEWAY_NAME=my-vm-gateway mise run gateway:vm
# OPENSHELL_SANDBOX_NAMESPACE=my-ns mise run gateway:vm
# OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:vm
# mise run gateway:vm -- --gpu
#
# This script also writes ~/.config/openshell/active_gateway so the
# `openshell` CLI automatically targets this gateway in subsequent shells.
@@ -84,21 +84,27 @@ register_gateway_metadata() {
local name=$1
local endpoint=$2
local port=$3
local vm_driver_state_dir=$4
local rootfs_artifact_secret=$5
local config_home gateway_dir
config_home="${XDG_CONFIG_HOME:-${HOME}/.config}"
gateway_dir="${config_home}/openshell/gateways/${name}"
mkdir -p "${gateway_dir}"
chmod 700 "${gateway_dir}" 2>/dev/null || true
cat >"${gateway_dir}/metadata.json" <<EOF
{
"name": "${name}",
"gateway_endpoint": "${endpoint}",
"is_remote": false,
"gateway_port": ${port},
"auth_mode": "plaintext"
"auth_mode": "plaintext",
"vm_driver_state_dir": "${vm_driver_state_dir}",
"vm_rootfs_artifact_secret": "${rootfs_artifact_secret}"
}
EOF
chmod 600 "${gateway_dir}/metadata.json" 2>/dev/null || true
}
# Mirror what `openshell gateway select <name>` does: write the gateway name
@@ -146,6 +152,51 @@ check_supervisor_cross_toolchain() {
fi
}
generate_rootfs_artifact_secret() {
od -An -N32 -tx1 /dev/urandom | tr -d ' \n'
}
VM_GPU="$(normalize_bool "${OPENSHELL_VM_GPU:-false}")"
while [ "$#" -gt 0 ]; do
case "$1" in
--gpu)
VM_GPU="true"
shift
;;
--gpu-mem-mib)
if [ "$#" -lt 2 ]; then
echo "ERROR: --gpu-mem-mib requires a value" >&2
exit 2
fi
export OPENSHELL_VM_GPU_MEM_MIB="$2"
shift 2
;;
--gpu-vcpus)
if [ "$#" -lt 2 ]; then
echo "ERROR: --gpu-vcpus requires a value" >&2
exit 2
fi
export OPENSHELL_VM_GPU_VCPUS="$2"
shift 2
;;
-h|--help)
echo "Usage: mise run gateway:vm -- [--gpu] [--gpu-mem-mib MIB] [--gpu-vcpus N]"
exit 0
;;
*)
echo "ERROR: unknown gateway-vm option '$1'" >&2
exit 2
;;
esac
done
if [ "${VM_GPU}" = "true" ]; then
export OPENSHELL_VM_GPU="true"
else
unset OPENSHELL_VM_GPU
fi
if [[ ! "${GATEWAY_NAME}" =~ ^[A-Za-z0-9._-]+$ ]]; then
echo "ERROR: OPENSHELL_VM_GATEWAY_NAME must contain only letters, numbers, dots, underscores, or dashes" >&2
exit 2
@@ -168,6 +219,12 @@ VM_DRIVER_STATE_DIR_DEFAULT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}/openshell-v
VM_DRIVER_STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${VM_DRIVER_STATE_DIR_DEFAULT}}"
DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")"
ROOTFS_ARTIFACT_SECRET="${OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET:-$(generate_rootfs_artifact_secret)}"
if [[ ! "${ROOTFS_ARTIFACT_SECRET}" =~ ^[A-Za-z0-9._~=-]+$ ]]; then
echo "ERROR: OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET must contain only URL-safe characters" >&2
exit 2
fi
export OPENSHELL_VM_ROOTFS_ARTIFACT_SECRET="${ROOTFS_ARTIFACT_SECRET}"
# Build prerequisites: VM runtime artifacts + bundled supervisor.
if [ ! -d "${COMPRESSED_DIR}" ] \
@@ -207,7 +264,7 @@ mkdir -p "${STATE_DIR}"
mkdir -p "${VM_DRIVER_STATE_DIR}"
GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}"
register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}"
register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}" "${ROOTFS_ARTIFACT_SECRET}"
save_active_gateway "${GATEWAY_NAME}"
echo "Starting standalone VM gateway..."
@@ -217,6 +274,7 @@ echo " namespace: ${SANDBOX_NAMESPACE}"
echo " state dir: ${STATE_DIR}"
echo " driver: ${DRIVER_DIR}/openshell-driver-vm"
echo " driver dir: ${VM_DRIVER_STATE_DIR}"
echo " gpu: ${VM_GPU}"
echo " image: ${SANDBOX_IMAGE}"
echo
echo "Active gateway set to '${GATEWAY_NAME}'. The CLI now targets this gateway"