trending/OpenShell
1
0
Fork 0
mirror of https://github.com/NVIDIA/OpenShell.git synced 2026-11-07 08:55:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(vm): add openshell-vm crate with libkrun microVM gateway (#611)

Browse Source
This commit is contained in:
Drew Newberry
2026-04-08 22:00:01 -07:00
committed by GitHub
parent c2e5256725
commit ddb85b1704
56 changed files with 10972 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/openshell-vm/src/embedded.rs
+442
View File
@@ -0,0 +1,442 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Embedded VM runtime resources.
//!
//! Native libraries (libkrun, libkrunfw, gvproxy) and the rootfs are embedded as
//! zstd-compressed byte arrays and extracted to XDG cache directories on first use.
//!
//! Cache locations:
//! - Runtime: `~/.local/share/openshell/vm-runtime/{version}/`
//! - Rootfs: `~/.local/share/openshell/openshell-vm/{version}/instances/<name>/rootfs/`
use std::fs;
use std::io::{Read, Write};
use std::path::{Path, PathBuf};
use indicatif::{ProgressBar, ProgressStyle};
use crate::VmError;
// ── Platform-specific embedded resources ───────────────────────────────────
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
mod resources {
pub const LIBKRUN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrun.dylib.zst"));
pub const LIBKRUNFW: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrunfw.5.dylib.zst"));
pub const GVPROXY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/gvproxy.zst"));
pub const ROOTFS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rootfs.tar.zst"));
pub const LIBKRUN_NAME: &str = "libkrun.dylib";
pub const LIBKRUNFW_NAME: &str = "libkrunfw.5.dylib";
}
#[cfg(all(target_os = "linux", target_arch = "aarch64"))]
mod resources {
pub const LIBKRUN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrun.so.zst"));
pub const LIBKRUNFW: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrunfw.so.5.zst"));
pub const GVPROXY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/gvproxy.zst"));
pub const ROOTFS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rootfs.tar.zst"));
pub const LIBKRUN_NAME: &str = "libkrun.so";
pub const LIBKRUNFW_NAME: &str = "libkrunfw.so.5";
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
mod resources {
pub const LIBKRUN: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrun.so.zst"));
pub const LIBKRUNFW: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/libkrunfw.so.5.zst"));
pub const GVPROXY: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/gvproxy.zst"));
pub const ROOTFS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rootfs.tar.zst"));
pub const LIBKRUN_NAME: &str = "libkrun.so";
pub const LIBKRUNFW_NAME: &str = "libkrunfw.so.5";
}
// Fallback for unsupported platforms (will fail at runtime)
#[cfg(not(any(
all(target_os = "macos", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64"),
)))]
mod resources {
pub const LIBKRUN: &[u8] = &[];
pub const LIBKRUNFW: &[u8] = &[];
pub const GVPROXY: &[u8] = &[];
pub const ROOTFS: &[u8] = &[];
pub const LIBKRUN_NAME: &str = "libkrun";
pub const LIBKRUNFW_NAME: &str = "libkrunfw";
}
const VERSION: &str = env!("CARGO_PKG_VERSION");
// ── Public API ─────────────────────────────────────────────────────────────
/// Ensures the embedded VM runtime is extracted to the cache directory.
///
/// Returns the path to the runtime directory containing:
/// - libkrun.{dylib,so}
/// - libkrunfw.{5.dylib,.so.5}
/// - gvproxy
///
/// On first call, this extracts the compressed embedded resources to the cache.
/// Subsequent calls return the cached path if valid.
pub fn ensure_runtime_extracted() -> Result<PathBuf, VmError> {
// Check if embedded resources are available (non-empty)
if resources::LIBKRUN.is_empty() {
return Err(VmError::HostSetup(
"VM runtime not embedded for this platform. \
Supported: macOS ARM64, Linux ARM64, Linux x86_64"
.to_string(),
));
}
let cache_dir = runtime_cache_dir()?;
let version_marker = cache_dir.join(".version");
// Cache key: version + content fingerprint (so dev builds at 0.0.0
// still invalidate when the embedded libraries change).
let cache_key = runtime_cache_key();
// Check if already extracted with the correct cache key
if version_marker.exists()
&& let Ok(cached_key) = fs::read_to_string(&version_marker)
&& cached_key.trim() == cache_key
{
// Validate files exist
if validate_runtime_dir(&cache_dir).is_ok() {
tracing::debug!(
path = %cache_dir.display(),
"Using cached VM runtime"
);
return Ok(cache_dir);
}
}
// Clean up old versions before extracting new one
cleanup_old_versions(&cache_dir)?;
// Create fresh directory
if cache_dir.exists() {
fs::remove_dir_all(&cache_dir)
.map_err(|e| VmError::HostSetup(format!("remove old cache: {e}")))?;
}
fs::create_dir_all(&cache_dir)
.map_err(|e| VmError::HostSetup(format!("create cache dir: {e}")))?;
tracing::info!(
path = %cache_dir.display(),
version = VERSION,
"Extracting embedded VM runtime"
);
// Extract all resources
extract_resource(resources::LIBKRUN, &cache_dir.join(resources::LIBKRUN_NAME))?;
extract_resource(
resources::LIBKRUNFW,
&cache_dir.join(resources::LIBKRUNFW_NAME),
)?;
extract_resource(resources::GVPROXY, &cache_dir.join("gvproxy"))?;
// Make gvproxy executable
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(cache_dir.join("gvproxy"), fs::Permissions::from_mode(0o755))
.map_err(|e| VmError::HostSetup(format!("chmod gvproxy: {e}")))?;
}
// Write version marker (includes content fingerprint for cache invalidation)
fs::write(&version_marker, runtime_cache_key())
.map_err(|e| VmError::HostSetup(format!("write version marker: {e}")))?;
tracing::info!(
path = %cache_dir.display(),
"VM runtime extracted successfully"
);
Ok(cache_dir)
}
/// Returns the path where the runtime would be cached (without extracting).
pub fn runtime_cache_path() -> Result<PathBuf, VmError> {
runtime_cache_dir()
}
/// Extract the embedded rootfs to the given destination directory.
///
/// If the destination already exists, it is returned as-is (no re-extraction).
/// Otherwise the embedded `rootfs.tar.zst` is decompressed and unpacked into `dest`.
///
/// A `.version` marker is written after successful extraction so that
/// version-mismatched rootfs directories are detected and rebuilt.
pub fn extract_rootfs_to(dest: &Path) -> Result<(), VmError> {
if resources::ROOTFS.is_empty() {
return Err(VmError::HostSetup(
"Rootfs not embedded. Build with: mise run vm:build:embedded".to_string(),
));
}
let version_marker = dest.join(".version");
// Already extracted with the correct version — nothing to do.
if version_marker.exists()
&& let Ok(cached_version) = fs::read_to_string(&version_marker)
&& cached_version.trim() == VERSION
{
tracing::debug!(
path = %dest.display(),
"Using cached rootfs"
);
return Ok(());
}
// Remove existing if present (version mismatch or incomplete extraction).
if dest.exists() {
eprintln!("Removing outdated rootfs at {}...", dest.display());
fs::remove_dir_all(dest)
.map_err(|e| VmError::HostSetup(format!("remove old rootfs: {e}")))?;
}
// Extract with progress bar.
extract_rootfs_with_progress(resources::ROOTFS, dest)?;
// Write version marker.
fs::write(&version_marker, VERSION)
.map_err(|e| VmError::HostSetup(format!("write rootfs version marker: {e}")))?;
Ok(())
}
/// Clean up rootfs directories from older versions.
///
/// Call this periodically (e.g. at startup) to reclaim disk from previous
/// releases. Removes all version directories under the openshell-vm base
/// except the current version.
pub fn cleanup_old_rootfs() -> Result<(), VmError> {
let base = rootfs_cache_base()?;
if !base.exists() {
return Ok(());
}
let current_version_dir = base.join(VERSION);
cleanup_old_versions_in_base(&base, &current_version_dir)
}
/// Check if the rootfs is embedded (non-empty).
pub fn has_embedded_rootfs() -> bool {
!resources::ROOTFS.is_empty()
}
// ── Internal helpers ───────────────────────────────────────────────────────
/// Build a cache key that combines the version string with a short content
/// fingerprint of the embedded runtime bytes.
///
/// Using the version alone is insufficient for dev builds (all `0.0.0`)
/// because the embedded libraries can change between compiles without the
/// version changing. The fingerprint is a simple XOR-fold of the first few
/// bytes of each embedded resource — cheap to compute at startup without
/// pulling in a hash dependency.
fn runtime_cache_key() -> String {
// XOR-fold the first 64 bytes of each resource to get a cheap fingerprint.
let mut fp: u64 = 0;
for (i, chunk) in [resources::LIBKRUN, resources::LIBKRUNFW, resources::GVPROXY]
.iter()
.enumerate()
{
let sample = &chunk[..chunk.len().min(64)];
let mut word: u64 = 0;
for (j, &b) in sample.iter().enumerate() {
word ^= (b as u64) << ((j % 8) * 8);
}
// Mix in resource index so identical resources don't cancel out.
fp ^= word.rotate_left((i as u32) * 13 + 7);
// Also mix in the total length so size changes are detected.
fp ^= (chunk.len() as u64).rotate_left((i as u32) * 17 + 3);
}
format!("{VERSION}-{fp:016x}")
}
fn runtime_cache_dir() -> Result<PathBuf, VmError> {
let base = openshell_core::paths::xdg_data_dir()
.map_err(|e| VmError::HostSetup(format!("resolve XDG data dir: {e}")))?;
Ok(base.join("openshell").join("vm-runtime").join(VERSION))
}
fn runtime_cache_base() -> Result<PathBuf, VmError> {
let base = openshell_core::paths::xdg_data_dir()
.map_err(|e| VmError::HostSetup(format!("resolve XDG data dir: {e}")))?;
Ok(base.join("openshell").join("vm-runtime"))
}
fn rootfs_cache_base() -> Result<PathBuf, VmError> {
let base = openshell_core::paths::xdg_data_dir()
.map_err(|e| VmError::HostSetup(format!("resolve XDG data dir: {e}")))?;
Ok(base.join("openshell").join("openshell-vm"))
}
fn cleanup_old_versions(current_dir: &Path) -> Result<(), VmError> {
cleanup_old_versions_in_base(&runtime_cache_base()?, current_dir)
}
fn cleanup_old_versions_in_base(base: &Path, current_dir: &Path) -> Result<(), VmError> {
if !base.exists() {
return Ok(());
}
let entries = match fs::read_dir(base) {
Ok(e) => e,
Err(_) => return Ok(()), // Can't read, skip cleanup
};
for entry in entries.filter_map(Result::ok) {
let path = entry.path();
// Skip if this is the current version directory or a parent of it
if path.is_dir() && !current_dir.starts_with(&path) && path != current_dir {
tracing::debug!(
path = %path.display(),
"Cleaning up old version"
);
if let Err(e) = fs::remove_dir_all(&path) {
tracing::warn!(
path = %path.display(),
error = %e,
"Failed to clean up old version"
);
}
}
}
Ok(())
}
fn extract_resource(compressed: &[u8], dest: &Path) -> Result<(), VmError> {
if compressed.is_empty() {
return Err(VmError::HostSetup(format!(
"embedded resource is empty: {}",
dest.display()
)));
}
let decompressed = zstd::decode_all(compressed)
.map_err(|e| VmError::HostSetup(format!("decompress {}: {e}", dest.display())))?;
let mut file = fs::File::create(dest)
.map_err(|e| VmError::HostSetup(format!("create {}: {e}", dest.display())))?;
file.write_all(&decompressed)
.map_err(|e| VmError::HostSetup(format!("write {}: {e}", dest.display())))?;
tracing::debug!(
path = %dest.display(),
compressed_size = compressed.len(),
decompressed_size = decompressed.len(),
"Extracted resource"
);
Ok(())
}
fn extract_rootfs_with_progress(compressed: &[u8], dest: &Path) -> Result<(), VmError> {
eprintln!("Extracting VM environment (first run)...");
// Create progress bar for decompression
let pb = ProgressBar::new(compressed.len() as u64);
pb.set_style(
ProgressStyle::default_bar()
.template(" Decompressing [{bar:40.cyan/blue}] {bytes}/{total_bytes}")
.unwrap()
.progress_chars("=>-"),
);
// Wrap the compressed data in a progress reader
let reader = ProgressReader::new(std::io::Cursor::new(compressed), pb.clone());
// Decompress zstd stream
let decoder = zstd::Decoder::new(reader)
.map_err(|e| VmError::HostSetup(format!("create zstd decoder: {e}")))?;
pb.finish_and_clear();
// Create destination directory
fs::create_dir_all(dest).map_err(|e| VmError::HostSetup(format!("create rootfs dir: {e}")))?;
// Extract tar archive with progress
eprintln!(" Extracting rootfs...");
let mut archive = tar::Archive::new(decoder);
archive
.unpack(dest)
.map_err(|e| VmError::HostSetup(format!("extract rootfs tarball: {e}")))?;
eprintln!(" Rootfs extracted to {}", dest.display());
Ok(())
}
/// A reader wrapper that updates a progress bar as data is read.
struct ProgressReader<R> {
inner: R,
progress: ProgressBar,
}
impl<R> ProgressReader<R> {
fn new(inner: R, progress: ProgressBar) -> Self {
Self { inner, progress }
}
}
impl<R: Read> Read for ProgressReader<R> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read(buf)?;
self.progress.inc(n as u64);
Ok(n)
}
}
fn validate_runtime_dir(dir: &Path) -> Result<(), VmError> {
let libkrun = dir.join(resources::LIBKRUN_NAME);
let libkrunfw = dir.join(resources::LIBKRUNFW_NAME);
let gvproxy = dir.join("gvproxy");
for path in [&libkrun, &libkrunfw, &gvproxy] {
if !path.exists() {
return Err(VmError::HostSetup(format!(
"missing runtime file: {}",
path.display()
)));
}
// Check file is not empty (would indicate a stub)
let size = fs::metadata(path).map(|m| m.len()).unwrap_or(0);
if size == 0 {
return Err(VmError::HostSetup(format!(
"runtime file is empty (stub): {}",
path.display()
)));
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_resources_not_empty() {
// On supported platforms, resources should be non-empty
#[cfg(any(
all(target_os = "macos", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64"),
))]
{
// Note: This test only passes if `mise run vm:setup` was run
// before building. In CI without compressed artifacts, resources will be
// empty stubs.
if !resources::LIBKRUN.is_empty() {
assert!(!resources::LIBKRUNFW.is_empty());
assert!(!resources::GVPROXY.is_empty());
}
}
}
}
crates/openshell-vm/src/exec.rs
+767
View File
@@ -0,0 +1,767 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use std::fs::{self, File};
use std::io::{BufRead, BufReader, Read, Write};
use std::os::unix::net::UnixStream;
use std::path::{Path, PathBuf};
use std::thread;
use std::time::{SystemTime, UNIX_EPOCH};
use base64::Engine as _;
use serde::{Deserialize, Serialize};
use crate::VmError;
/// Remove a directory, safely handling symlinks.
///
/// Uses `symlink_metadata` (lstat) to detect symlinks. If the path is a
/// symlink (e.g. `var/run -> /run` in a Linux rootfs), the symlink itself
/// is removed without following it — preventing traversal attacks where a
/// symlink could redirect `remove_dir_all` to an arbitrary host path.
/// If the path is a real directory, it is removed recursively.
fn safe_remove_dir_all(path: &Path) -> Result<bool, VmError> {
match fs::symlink_metadata(path) {
Ok(meta) => {
if meta.file_type().is_symlink() {
// Remove the symlink itself, not the target it points to.
fs::remove_file(path).map_err(|e| {
VmError::RuntimeState(format!("reset: remove symlink {}: {e}", path.display()))
})?;
return Ok(true);
}
if !meta.is_dir() {
return Ok(false); // Not a directory — nothing to remove.
}
fs::remove_dir_all(path).map_err(|e| {
VmError::RuntimeState(format!("reset: remove {}: {e}", path.display()))
})?;
Ok(true)
}
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(false),
Err(e) => Err(VmError::RuntimeState(format!(
"stat {}: {e}",
path.display()
))),
}
}
pub const VM_EXEC_VSOCK_PORT: u32 = 10_777;
const VM_STATE_NAME: &str = "vm-state.json";
const VM_LOCK_NAME: &str = "vm.lock";
const KUBECONFIG_ENV: &str = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml";
#[derive(Debug, Clone)]
pub struct VmExecOptions {
pub rootfs: Option<PathBuf>,
pub command: Vec<String>,
pub workdir: Option<String>,
pub env: Vec<String>,
pub tty: bool,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct VmRuntimeState {
pub pid: i32,
pub exec_vsock_port: u32,
pub socket_path: PathBuf,
pub rootfs: PathBuf,
pub console_log: PathBuf,
pub started_at_ms: u128,
/// PID of the gvproxy process (if networking uses gvproxy).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gvproxy_pid: Option<u32>,
}
#[derive(Debug, Serialize)]
struct ExecRequest {
argv: Vec<String>,
env: Vec<String>,
cwd: Option<String>,
tty: bool,
}
#[derive(Debug, Serialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum ClientFrame {
Stdin { data: String },
StdinClose,
}
#[derive(Debug, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
enum ServerFrame {
Stdout { data: String },
Stderr { data: String },
Exit { code: i32 },
Error { message: String },
}
pub fn vm_exec_socket_path(rootfs: &Path) -> PathBuf {
// Prefer XDG_RUNTIME_DIR (per-user, restricted permissions on Linux),
// fall back to /tmp. Ownership/symlink validation happens in
// secure_socket_base() when the gvproxy socket dir is created; here
// we just compute the path. The parent directory is created (with
// permission checks) at launch time via create_dir_all.
let base = if let Some(xdg) = std::env::var_os("XDG_RUNTIME_DIR") {
PathBuf::from(xdg)
} else {
let mut base = PathBuf::from("/tmp");
if !base.is_dir() {
base = std::env::temp_dir();
}
base
};
let dir = base.join("ovm-exec");
let id = hash_path_id(rootfs);
dir.join(format!("{id}.sock"))
}
fn hash_path_id(path: &Path) -> String {
let mut hash: u64 = 0xcbf29ce484222325;
for byte in path.to_string_lossy().as_bytes() {
hash ^= u64::from(*byte);
hash = hash.wrapping_mul(0x100000001b3);
}
format!("{:012x}", hash & 0x0000_ffff_ffff_ffff)
}
pub fn write_vm_runtime_state(
rootfs: &Path,
pid: i32,
console_log: &Path,
gvproxy_pid: Option<u32>,
) -> Result<(), VmError> {
let state = VmRuntimeState {
pid,
exec_vsock_port: VM_EXEC_VSOCK_PORT,
socket_path: vm_exec_socket_path(rootfs),
rootfs: rootfs.to_path_buf(),
console_log: console_log.to_path_buf(),
started_at_ms: now_ms()?,
gvproxy_pid,
};
let path = vm_state_path(rootfs);
let bytes = serde_json::to_vec_pretty(&state)
.map_err(|e| VmError::RuntimeState(format!("serialize VM runtime state: {e}")))?;
fs::create_dir_all(vm_run_dir(rootfs))
.map_err(|e| VmError::RuntimeState(format!("create VM runtime dir: {e}")))?;
fs::write(&path, bytes)
.map_err(|e| VmError::RuntimeState(format!("write {}: {e}", path.display())))?;
Ok(())
}
pub fn clear_vm_runtime_state(rootfs: &Path) {
let state_path = vm_state_path(rootfs);
let socket_path = vm_exec_socket_path(rootfs);
let _ = fs::remove_file(state_path);
let _ = fs::remove_file(socket_path);
}
/// Wipe stale container runtime state from the rootfs.
///
/// After a crash or unclean shutdown, containerd and kubelet can retain
/// references to pod sandboxes and containers that no longer exist. This
/// causes `ContainerCreating` → `context deadline exceeded` loops because
/// containerd blocks trying to clean up orphaned resources.
///
/// This function removes:
/// - containerd runtime task state (running container metadata)
/// - containerd sandbox controller shim state
/// - containerd CRI plugin state (pod/container tracking)
/// - containerd tmp mounts
/// - kubelet pod state (volume mounts, pod status)
///
/// It preserves:
/// - containerd images and content (no re-pull needed)
/// - containerd snapshots (no re-extract needed)
/// - containerd metadata database (meta.db — image/snapshot tracking)
///
/// **Note:** This is the only path that wipes the kine/SQLite database.
/// Normal boots preserve `state.db` (and all cluster objects) across
/// restarts. The init script clears stale bootstrap locks via `sqlite3`,
/// and `recover_corrupt_kine_db` handles actual file corruption.
pub fn reset_runtime_state(rootfs: &Path, gateway_name: &str) -> Result<(), VmError> {
// Full reset: wipe all runtime state so the VM cold-starts from scratch.
//
// With the block-device layout, k3s server/agent state, containerd, PVCs,
// and PKI all live on the state disk — the caller in lib.rs deletes the
// entire state disk image file, which achieves a complete wipe in one
// operation without touching the virtiofs rootfs.
//
// We still clean the virtiofs rootfs for paths that are NOT on the state
// disk: kubelet pod volumes, CNI state, and the pre-init sentinel. These
// paths are present in the rootfs regardless of the storage layout.
let dirs_to_remove = [
// Stale pod volume mounts and projected secrets
rootfs.join("var/lib/kubelet/pods"),
// CNI state: stale network namespace references from dead pods
rootfs.join("var/lib/cni"),
// Runtime state (PIDs, sockets) — on virtiofs, not block device
rootfs.join("var/run"),
];
let mut cleaned = 0usize;
for dir in &dirs_to_remove {
if safe_remove_dir_all(dir)? {
cleaned += 1;
}
}
// Remove the pre-initialized sentinel so the init script knows
// this is a cold start and deploys manifests from staging.
// We write a marker file so ensure-vm-rootfs.sh still sees the
// rootfs as built (avoiding a full rebuild) while the init script
// detects the cold start via the missing .initialized sentinel.
let sentinel = rootfs.join("opt/openshell/.initialized");
let reset_marker = rootfs.join("opt/openshell/.reset");
if sentinel.exists() {
fs::remove_file(&sentinel).map_err(|e| {
VmError::RuntimeState(format!(
"reset: remove sentinel {}: {e}",
sentinel.display()
))
})?;
fs::write(&reset_marker, "").map_err(|e| {
VmError::RuntimeState(format!(
"reset: write marker {}: {e}",
reset_marker.display()
))
})?;
cleaned += 1;
}
// PKI lives on the state disk; deleting the state disk image (done by
// the caller) rotates it automatically. Just note it for the log.
eprintln!("Reset: PKI will be regenerated on next boot (state disk wiped)");
// Wipe host-side mTLS credentials so bootstrap_gateway() takes the
// first-boot path and fetches new certs from the VM via the exec agent.
if let Ok(home) = std::env::var("HOME") {
let config_base =
std::env::var("XDG_CONFIG_HOME").unwrap_or_else(|_| format!("{home}/.config"));
let mtls_dir = PathBuf::from(&config_base)
.join("openshell/gateways")
.join(gateway_name)
.join("mtls");
if mtls_dir.is_dir() {
fs::remove_dir_all(&mtls_dir).map_err(|e| {
VmError::RuntimeState(format!(
"reset: remove mTLS dir {}: {e}",
mtls_dir.display()
))
})?;
}
// Also remove metadata so is_warm_boot() returns false.
let metadata = PathBuf::from(&config_base)
.join("openshell/gateways")
.join(gateway_name)
.join("metadata.json");
if metadata.is_file() {
fs::remove_file(&metadata).map_err(|e| {
VmError::RuntimeState(format!(
"reset: remove metadata {}: {e}",
metadata.display()
))
})?;
}
}
eprintln!("Reset: cleaned {cleaned} state directories (full reset)");
Ok(())
}
/// Remove a corrupt kine (`SQLite`) database so k3s can recreate it on boot.
///
/// k3s uses kine with a `SQLite` backend at `var/lib/rancher/k3s/server/db/state.db`.
/// If the VM is killed mid-write (SIGKILL, host crash, power loss), the database
/// file may be left in a corrupt state — the `SQLite` header magic is missing or the
/// file is truncated. k3s would open the DB, get `SQLITE_NOTADB` /
/// `SQLITE_CORRUPT`, and crash at startup.
///
/// This function checks the `SQLite` file header (first 100 bytes only) and removes
/// the database plus its WAL/SHM sidecar files if the header is invalid. k3s will
/// create a fresh database on startup and cluster state will be re-applied from
/// the auto-deploy manifests in `server/manifests/`.
///
/// **Stale bootstrap locks** (a kine application-level issue where a killed k3s
/// server leaves a lock row that causes the next instance to hang) are handled
/// separately by the init script (`openshell-vm-init.sh`), which runs
/// `sqlite3 state.db "DELETE FROM kine WHERE name LIKE '/bootstrap/%'"` before
/// starting k3s. This allows the database — and all persistent cluster state — to
/// survive normal restarts.
///
/// **What is lost on corruption:** all cluster object records (Pods, Deployments,
/// Secrets, `ConfigMaps`, CRDs, etc.) and the bootstrap token. These are re-created
/// from manifests on the next boot.
///
/// **What is always preserved:** container images and snapshots (under
/// `k3s/agent/`), PKI, and the `.initialized` sentinel.
///
/// This function is a no-op if `state.db` does not exist (e.g. first boot or
/// after a full `--reset`).
pub fn recover_corrupt_kine_db(rootfs: &Path) -> Result<(), VmError> {
let db_path = rootfs.join("var/lib/rancher/k3s/server/db/state.db");
if !db_path.exists() {
return Ok(()); // Nothing to check — first boot or post-reset.
}
// The SQLite file format begins with a 16-byte magic string.
// Reference: https://www.sqlite.org/fileformat.html#the_database_header
const SQLITE_MAGIC: &[u8] = b"SQLite format 3\x00";
// Read only the first 100 bytes (the minimum valid SQLite header size)
// instead of loading the entire database into memory.
let has_invalid_header = match File::open(&db_path).and_then(|mut f| {
let mut buf = [0u8; 100];
let n = f.read(&mut buf)?;
Ok((n, buf))
}) {
Err(_) => true, // Can't read → treat as corrupt.
Ok((n, _)) if n < 100 => true, // Too short to be a valid DB.
Ok((_, buf)) => !buf.starts_with(SQLITE_MAGIC),
};
if !has_invalid_header {
return Ok(()); // Valid database — preserve it for warm boot.
}
eprintln!(
"Warning: kine database is corrupt ({}), removing for clean boot",
db_path.display()
);
remove_kine_db_files(&db_path)?;
Ok(())
}
/// Remove the kine `SQLite` database and its WAL/SHM sidecar files.
fn remove_kine_db_files(db_path: &Path) -> Result<(), VmError> {
if let Err(e) = fs::remove_file(db_path) {
return Err(VmError::RuntimeState(format!(
"failed to remove kine database {}: {e}",
db_path.display()
)));
}
// Also remove any WAL/SHM sidecar files left by an interrupted write.
let _ = fs::remove_file(db_path.with_extension("db-wal"));
let _ = fs::remove_file(db_path.with_extension("db-shm"));
Ok(())
}
/// Acquire an exclusive lock on the rootfs lock file.
///
/// The lock is held for the lifetime of the returned `File` handle. When
/// the process exits (even via SIGKILL), the OS releases the lock
/// automatically. This provides a reliable guard against two VM processes
/// sharing the same rootfs — even if the state file is deleted.
///
/// Returns `Ok(File)` on success. The caller must keep the `File` alive
/// for as long as the VM is running.
pub fn acquire_rootfs_lock(rootfs: &Path) -> Result<File, VmError> {
let lock_path = vm_lock_path(rootfs);
fs::create_dir_all(vm_run_dir(rootfs))
.map_err(|e| VmError::RuntimeState(format!("create VM runtime dir: {e}")))?;
// Open (or create) the lock file without truncating so we can read
// the holder's PID for the error message if the lock is held.
let file = fs::OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&lock_path)
.map_err(|e| {
VmError::RuntimeState(format!("open lock file {}: {e}", lock_path.display()))
})?;
// Try non-blocking exclusive lock.
let fd = std::os::unix::io::AsRawFd::as_raw_fd(&file);
let rc = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) };
if rc != 0 {
let err = std::io::Error::last_os_error();
if err.raw_os_error() == Some(libc::EWOULDBLOCK) {
// Another process holds the lock — read its PID for diagnostics.
let holder_pid = fs::read_to_string(&lock_path).unwrap_or_default();
let holder_pid = holder_pid.trim();
return Err(VmError::RuntimeState(format!(
"another process (pid {holder_pid}) is using rootfs {}. \
Stop the running VM first",
rootfs.display()
)));
}
return Err(VmError::RuntimeState(format!(
"lock rootfs {}: {err}",
lock_path.display()
)));
}
// Lock acquired — write our PID (truncate first, then write).
// This is informational only; the flock is the real guard.
let _ = file.set_len(0);
{
let mut f = &file;
let _ = write!(f, "{}", std::process::id());
}
Ok(file)
}
/// Check whether the rootfs lock file is currently held by another process.
///
/// Returns `Ok(())` if the lock is free (or can be acquired), and an
/// `Err` if another process holds it. Does NOT acquire the lock — use
/// [`acquire_rootfs_lock`] for that.
fn check_rootfs_lock_free(rootfs: &Path) -> Result<(), VmError> {
let lock_path = vm_lock_path(rootfs);
if !lock_path.exists() {
return Ok(());
}
let Ok(file) = File::open(&lock_path) else {
return Ok(()); // Can't open → treat as free
};
let fd = std::os::unix::io::AsRawFd::as_raw_fd(&file);
let rc = unsafe { libc::flock(fd, libc::LOCK_EX | libc::LOCK_NB) };
if rc != 0 {
let err = std::io::Error::last_os_error();
if err.raw_os_error() == Some(libc::EWOULDBLOCK) {
let holder_pid = fs::read_to_string(&lock_path).unwrap_or_default();
let holder_pid = holder_pid.trim();
return Err(VmError::RuntimeState(format!(
"another process (pid {holder_pid}) is using rootfs {}. \
Stop the running VM first",
rootfs.display()
)));
}
} else {
// We acquired the lock — release it immediately since we're only probing.
unsafe { libc::flock(fd, libc::LOCK_UN) };
}
Ok(())
}
pub fn ensure_vm_not_running(rootfs: &Path) -> Result<(), VmError> {
// Primary guard: check the flock. This works even if the state file
// has been deleted, because the kernel holds the lock until the
// owning process exits.
check_rootfs_lock_free(rootfs)?;
// Secondary guard: check the state file for any stale state.
match load_vm_runtime_state(Some(rootfs)) {
Ok(state) => Err(VmError::RuntimeState(format!(
"VM is already running (pid {}) with exec socket {}",
state.pid,
state.socket_path.display()
))),
Err(VmError::RuntimeState(message))
if message.starts_with("read VM runtime state")
|| message.starts_with("VM is not running") =>
{
clear_vm_runtime_state(rootfs);
Ok(())
}
Err(err) => Err(err),
}
}
pub fn exec_running_vm(options: VmExecOptions) -> Result<i32, VmError> {
let state = load_vm_runtime_state(options.rootfs.as_deref())?;
let mut stream = UnixStream::connect(&state.socket_path).map_err(|e| {
VmError::Exec(format!(
"connect to VM exec socket {}: {e}",
state.socket_path.display()
))
})?;
let mut writer = stream
.try_clone()
.map_err(|e| VmError::Exec(format!("clone VM exec socket: {e}")))?;
let mut env = options.env;
validate_env_vars(&env)?;
if !env.iter().any(|item| item.starts_with("KUBECONFIG=")) {
env.push(KUBECONFIG_ENV.to_string());
}
let request = ExecRequest {
argv: options.command,
env,
cwd: options.workdir,
tty: options.tty,
};
send_json_line(&mut writer, &request)?;
let stdin_writer = writer;
thread::spawn(move || {
let _ = pump_stdin(stdin_writer);
});
let mut reader = BufReader::new(&mut stream);
let mut line = String::new();
let stdout = std::io::stdout();
let stderr = std::io::stderr();
let mut stdout = stdout.lock();
let mut stderr = stderr.lock();
let mut exit_code = None;
loop {
line.clear();
let bytes = reader
.read_line(&mut line)
.map_err(|e| VmError::Exec(format!("read VM exec response from guest agent: {e}")))?;
if bytes == 0 {
break;
}
let frame: ServerFrame = serde_json::from_str(line.trim_end())
.map_err(|e| VmError::Exec(format!("decode VM exec response frame: {e}")))?;
match frame {
ServerFrame::Stdout { data } => {
let bytes = decode_payload(&data)?;
stdout
.write_all(&bytes)
.map_err(|e| VmError::Exec(format!("write guest stdout: {e}")))?;
stdout
.flush()
.map_err(|e| VmError::Exec(format!("flush guest stdout: {e}")))?;
}
ServerFrame::Stderr { data } => {
let bytes = decode_payload(&data)?;
stderr
.write_all(&bytes)
.map_err(|e| VmError::Exec(format!("write guest stderr: {e}")))?;
stderr
.flush()
.map_err(|e| VmError::Exec(format!("flush guest stderr: {e}")))?;
}
ServerFrame::Exit { code } => {
exit_code = Some(code);
break;
}
ServerFrame::Error { message } => {
return Err(VmError::Exec(message));
}
}
}
exit_code.ok_or_else(|| {
VmError::Exec("VM exec agent disconnected before returning an exit code".to_string())
})
}
/// Run a command inside the guest via the exec agent and capture its stdout.
///
/// Unlike [`exec_running_vm`], this function does not pump host stdin or write
/// to the terminal. It collects all stdout frames into a `Vec<u8>` and returns
/// them on success (exit code 0). Stderr output is discarded.
///
/// This is the building block for internal host→guest queries (e.g. reading
/// files from the guest filesystem) without requiring a dedicated vsock server.
pub fn exec_capture(socket_path: &Path, argv: Vec<String>) -> Result<Vec<u8>, VmError> {
let mut stream = UnixStream::connect(socket_path).map_err(|e| {
VmError::Exec(format!(
"connect to VM exec socket {}: {e}",
socket_path.display()
))
})?;
let mut writer = stream
.try_clone()
.map_err(|e| VmError::Exec(format!("clone VM exec socket: {e}")))?;
let request = ExecRequest {
argv,
env: vec![],
cwd: None,
tty: false,
};
send_json_line(&mut writer, &request)?;
// Close stdin immediately — we have no input to send.
send_json_line(&mut writer, &ClientFrame::StdinClose)?;
let mut reader = BufReader::new(&mut stream);
let mut line = String::new();
let mut stdout_buf = Vec::new();
loop {
line.clear();
let bytes = reader
.read_line(&mut line)
.map_err(|e| VmError::Exec(format!("read VM exec response: {e}")))?;
if bytes == 0 {
break;
}
let frame: ServerFrame = serde_json::from_str(line.trim_end())
.map_err(|e| VmError::Exec(format!("decode VM exec response frame: {e}")))?;
match frame {
ServerFrame::Stdout { data } => {
stdout_buf.extend_from_slice(&decode_payload(&data)?);
}
ServerFrame::Stderr { .. } => {
// Discard stderr for capture mode.
}
ServerFrame::Exit { code } => {
if code != 0 {
return Err(VmError::Exec(format!(
"guest command exited with code {code}"
)));
}
return Ok(stdout_buf);
}
ServerFrame::Error { message } => {
return Err(VmError::Exec(message));
}
}
}
Err(VmError::Exec(
"VM exec agent disconnected before returning an exit code".to_string(),
))
}
fn vm_run_dir(rootfs: &Path) -> PathBuf {
rootfs.parent().unwrap_or(rootfs).to_path_buf()
}
pub fn vm_state_path(rootfs: &Path) -> PathBuf {
vm_run_dir(rootfs).join(format!("{}-{}", rootfs_key(rootfs), VM_STATE_NAME))
}
fn vm_lock_path(rootfs: &Path) -> PathBuf {
vm_run_dir(rootfs).join(format!("{}-{}", rootfs_key(rootfs), VM_LOCK_NAME))
}
fn rootfs_key(rootfs: &Path) -> String {
let name = rootfs
.file_name()
.and_then(|part| part.to_str())
.unwrap_or("openshell-vm");
let mut out = String::with_capacity(name.len());
for ch in name.chars() {
if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' {
out.push(ch);
} else {
out.push('_');
}
}
if out.is_empty() {
"openshell-vm".to_string()
} else {
out
}
}
fn default_rootfs() -> Result<PathBuf, VmError> {
crate::named_rootfs_dir("default")
}
fn load_vm_runtime_state(rootfs: Option<&Path>) -> Result<VmRuntimeState, VmError> {
let rootfs = match rootfs {
Some(rootfs) => rootfs.to_path_buf(),
None => default_rootfs()?,
};
let path = vm_state_path(&rootfs);
let bytes = fs::read(&path).map_err(|e| {
VmError::RuntimeState(format!(
"read VM runtime state {}: {e}. Start the VM with `openshell-vm` first",
path.display()
))
})?;
let state: VmRuntimeState = serde_json::from_slice(&bytes)
.map_err(|e| VmError::RuntimeState(format!("decode VM runtime state: {e}")))?;
if !process_alive(state.pid) {
clear_vm_runtime_state(&state.rootfs);
return Err(VmError::RuntimeState(format!(
"VM is not running (stale pid {})",
state.pid
)));
}
if !state.socket_path.exists() {
return Err(VmError::RuntimeState(format!(
"VM exec socket is not ready: {}",
state.socket_path.display()
)));
}
Ok(state)
}
fn validate_env_vars(items: &[String]) -> Result<(), VmError> {
for item in items {
let (key, _value) = item.split_once('=').ok_or_else(|| {
VmError::Exec(format!(
"invalid environment variable `{item}`; expected KEY=VALUE"
))
})?;
if key.is_empty()
|| !key.chars().enumerate().all(|(idx, ch)| {
ch == '_' || (ch.is_ascii_alphanumeric() && (idx > 0 || !ch.is_ascii_digit()))
})
{
return Err(VmError::Exec(format!(
"invalid environment variable name `{key}`"
)));
}
}
Ok(())
}
fn send_json_line<T: Serialize>(writer: &mut UnixStream, value: &T) -> Result<(), VmError> {
let mut bytes = serde_json::to_vec(value)
.map_err(|e| VmError::Exec(format!("encode VM exec request: {e}")))?;
bytes.push(b'\n');
writer
.write_all(&bytes)
.map_err(|e| VmError::Exec(format!("write VM exec request: {e}")))
}
fn pump_stdin(mut writer: UnixStream) -> Result<(), VmError> {
let stdin = std::io::stdin();
let mut stdin = stdin.lock();
let mut buf = [0u8; 8192];
loop {
let read = stdin
.read(&mut buf)
.map_err(|e| VmError::Exec(format!("read local stdin: {e}")))?;
if read == 0 {
break;
}
let frame = ClientFrame::Stdin {
data: base64::engine::general_purpose::STANDARD.encode(&buf[..read]),
};
send_json_line(&mut writer, &frame)?;
}
send_json_line(&mut writer, &ClientFrame::StdinClose)
}
fn decode_payload(data: &str) -> Result<Vec<u8>, VmError> {
base64::engine::general_purpose::STANDARD
.decode(data)
.map_err(|e| VmError::Exec(format!("decode VM exec payload: {e}")))
}
fn process_alive(pid: i32) -> bool {
let rc = unsafe { libc::kill(pid, 0) };
if rc == 0 {
return true;
}
std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM)
}
fn now_ms() -> Result<u128, VmError> {
let duration = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map_err(|e| VmError::RuntimeState(format!("read system clock: {e}")))?;
Ok(duration.as_millis())
}
crates/openshell-vm/src/ffi.rs
+336
View File
@@ -0,0 +1,336 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Minimal runtime-loaded bindings for the libkrun C API.
//!
//! We intentionally do not link libkrun at build time. Instead, the
//! `openshell-vm` binary loads `libkrun` from the staged `openshell-vm.runtime/`
//! sidecar bundle on first use.
use std::fs;
use std::path::{Path, PathBuf};
use std::sync::OnceLock;
use libc::c_char;
use libloading::Library;
use crate::VmError;
/// Runtime provenance information extracted from the bundle.
#[derive(Debug, Clone)]
pub struct RuntimeProvenance {
/// Path to the libkrun library that was loaded.
pub libkrun_path: PathBuf,
/// Paths to all libkrunfw libraries that were preloaded.
pub libkrunfw_paths: Vec<PathBuf>,
/// SHA-256 hash of the primary libkrunfw artifact (if computable).
pub libkrunfw_sha256: Option<String>,
/// Contents of provenance.json if present in the runtime bundle.
pub provenance_json: Option<String>,
/// Whether this is a custom (OpenShell-built) runtime.
pub is_custom: bool,
}
pub const KRUN_LOG_TARGET_DEFAULT: i32 = -1;
pub const KRUN_LOG_LEVEL_OFF: u32 = 0;
pub const KRUN_LOG_LEVEL_ERROR: u32 = 1;
pub const KRUN_LOG_LEVEL_WARN: u32 = 2;
pub const KRUN_LOG_LEVEL_INFO: u32 = 3;
pub const KRUN_LOG_LEVEL_DEBUG: u32 = 4;
pub const KRUN_LOG_LEVEL_TRACE: u32 = 5;
pub const KRUN_LOG_STYLE_AUTO: u32 = 0;
pub const KRUN_LOG_OPTION_NO_ENV: u32 = 1;
pub const KRUN_DISK_FORMAT_RAW: u32 = 0;
#[allow(dead_code)] // Used only on macOS (cfg-gated in state_disk_sync_mode)
pub const KRUN_SYNC_RELAXED: u32 = 1;
#[allow(dead_code)] // Used only on Linux (cfg-gated in state_disk_sync_mode)
pub const KRUN_SYNC_FULL: u32 = 2;
type KrunInitLog =
unsafe extern "C" fn(target_fd: i32, level: u32, style: u32, options: u32) -> i32;
type KrunCreateCtx = unsafe extern "C" fn() -> i32;
type KrunFreeCtx = unsafe extern "C" fn(ctx_id: u32) -> i32;
type KrunSetVmConfig = unsafe extern "C" fn(ctx_id: u32, num_vcpus: u8, ram_mib: u32) -> i32;
type KrunSetRoot = unsafe extern "C" fn(ctx_id: u32, root_path: *const c_char) -> i32;
type KrunSetWorkdir = unsafe extern "C" fn(ctx_id: u32, workdir_path: *const c_char) -> i32;
type KrunSetExec = unsafe extern "C" fn(
ctx_id: u32,
exec_path: *const c_char,
argv: *const *const c_char,
envp: *const *const c_char,
) -> i32;
type KrunSetPortMap = unsafe extern "C" fn(ctx_id: u32, port_map: *const *const c_char) -> i32;
type KrunSetConsoleOutput = unsafe extern "C" fn(ctx_id: u32, filepath: *const c_char) -> i32;
type KrunAddDisk3 = unsafe extern "C" fn(
ctx_id: u32,
block_id: *const c_char,
disk_path: *const c_char,
disk_format: u32,
read_only: bool,
direct_io: bool,
sync_mode: u32,
) -> i32;
type KrunAddVsockPort2 =
unsafe extern "C" fn(ctx_id: u32, port: u32, c_filepath: *const c_char, listen: bool) -> i32;
type KrunStartEnter = unsafe extern "C" fn(ctx_id: u32) -> i32;
type KrunDisableImplicitVsock = unsafe extern "C" fn(ctx_id: u32) -> i32;
type KrunAddVsock = unsafe extern "C" fn(ctx_id: u32, tsi_features: u32) -> i32;
#[cfg(target_os = "macos")]
type KrunAddNetUnixgram = unsafe extern "C" fn(
ctx_id: u32,
c_path: *const c_char,
fd: i32,
c_mac: *const u8,
features: u32,
flags: u32,
) -> i32;
type KrunAddNetUnixstream = unsafe extern "C" fn(
ctx_id: u32,
c_path: *const c_char,
fd: i32,
c_mac: *const u8,
features: u32,
flags: u32,
) -> i32;
pub struct LibKrun {
pub krun_init_log: KrunInitLog,
pub krun_create_ctx: KrunCreateCtx,
pub krun_free_ctx: KrunFreeCtx,
pub krun_set_vm_config: KrunSetVmConfig,
pub krun_set_root: KrunSetRoot,
pub krun_set_workdir: KrunSetWorkdir,
pub krun_set_exec: KrunSetExec,
pub krun_set_port_map: KrunSetPortMap,
pub krun_set_console_output: KrunSetConsoleOutput,
pub krun_add_disk3: Option<KrunAddDisk3>,
pub krun_add_vsock_port2: KrunAddVsockPort2,
pub krun_start_enter: KrunStartEnter,
pub krun_disable_implicit_vsock: KrunDisableImplicitVsock,
pub krun_add_vsock: KrunAddVsock,
#[cfg(target_os = "macos")]
pub krun_add_net_unixgram: KrunAddNetUnixgram,
#[allow(dead_code)] // FFI symbol loaded for future use
pub krun_add_net_unixstream: KrunAddNetUnixstream,
}
static LIBKRUN: OnceLock<LibKrun> = OnceLock::new();
static RUNTIME_PROVENANCE: OnceLock<RuntimeProvenance> = OnceLock::new();
pub fn libkrun() -> Result<&'static LibKrun, VmError> {
if let Some(lib) = LIBKRUN.get() {
return Ok(lib);
}
let loaded = LibKrun::load()?;
let _ = LIBKRUN.set(loaded);
Ok(LIBKRUN.get().expect("libkrun should be initialized"))
}
/// Return the provenance information for the loaded runtime.
///
/// Only available after [`libkrun()`] has been called successfully.
pub fn runtime_provenance() -> Option<&'static RuntimeProvenance> {
RUNTIME_PROVENANCE.get()
}
impl LibKrun {
fn load() -> Result<Self, VmError> {
let path = runtime_libkrun_path()?;
let runtime_dir = path.parent().ok_or_else(|| {
VmError::HostSetup(format!("libkrun has no parent dir: {}", path.display()))
})?;
let krunfw_paths = preload_runtime_support_libraries(runtime_dir)?;
// Build and store provenance information.
let provenance_json_path = runtime_dir.join("provenance.json");
let provenance_json = fs::read_to_string(&provenance_json_path).ok();
let is_custom = provenance_json.is_some();
let libkrunfw_sha256 = krunfw_paths.first().and_then(|p| compute_sha256(p).ok());
let provenance = RuntimeProvenance {
libkrun_path: path.clone(),
libkrunfw_paths: krunfw_paths,
libkrunfw_sha256,
provenance_json,
is_custom,
};
let _ = RUNTIME_PROVENANCE.set(provenance);
let library = Box::leak(Box::new(unsafe {
Library::new(&path).map_err(|e| {
VmError::HostSetup(format!("load libkrun from {}: {e}", path.display()))
})?
}));
Ok(Self {
krun_init_log: load_symbol(library, b"krun_init_log\0", &path)?,
krun_create_ctx: load_symbol(library, b"krun_create_ctx\0", &path)?,
krun_free_ctx: load_symbol(library, b"krun_free_ctx\0", &path)?,
krun_set_vm_config: load_symbol(library, b"krun_set_vm_config\0", &path)?,
krun_set_root: load_symbol(library, b"krun_set_root\0", &path)?,
krun_set_workdir: load_symbol(library, b"krun_set_workdir\0", &path)?,
krun_set_exec: load_symbol(library, b"krun_set_exec\0", &path)?,
krun_set_port_map: load_symbol(library, b"krun_set_port_map\0", &path)?,
krun_set_console_output: load_symbol(library, b"krun_set_console_output\0", &path)?,
krun_add_disk3: load_optional_symbol(library, b"krun_add_disk3\0"),
krun_add_vsock_port2: load_symbol(library, b"krun_add_vsock_port2\0", &path)?,
krun_start_enter: load_symbol(library, b"krun_start_enter\0", &path)?,
krun_disable_implicit_vsock: load_symbol(
library,
b"krun_disable_implicit_vsock\0",
&path,
)?,
krun_add_vsock: load_symbol(library, b"krun_add_vsock\0", &path)?,
#[cfg(target_os = "macos")]
krun_add_net_unixgram: load_symbol(library, b"krun_add_net_unixgram\0", &path)?,
krun_add_net_unixstream: load_symbol(library, b"krun_add_net_unixstream\0", &path)?,
})
}
}
fn runtime_libkrun_path() -> Result<PathBuf, VmError> {
Ok(crate::configured_runtime_dir()?.join(required_runtime_lib_name()))
}
fn preload_runtime_support_libraries(runtime_dir: &Path) -> Result<Vec<PathBuf>, VmError> {
let entries = fs::read_dir(runtime_dir)
.map_err(|e| VmError::HostSetup(format!("read {}: {e}", runtime_dir.display())))?;
let mut support_libs: Vec<PathBuf> = entries
.filter_map(Result::ok)
.map(|entry| entry.path())
.filter(|path| {
path.file_name()
.and_then(|name| name.to_str())
.is_some_and(|name| {
#[cfg(target_os = "macos")]
{
name.starts_with("libkrunfw") && name.ends_with(".dylib")
}
#[cfg(not(target_os = "macos"))]
{
name.starts_with("libkrunfw") && name.contains(".so")
}
})
})
.collect();
support_libs.sort();
for path in &support_libs {
let path_cstr = std::ffi::CString::new(path.to_string_lossy().as_bytes()).map_err(|e| {
VmError::HostSetup(format!(
"invalid support library path {}: {e}",
path.display()
))
})?;
let handle =
unsafe { libc::dlopen(path_cstr.as_ptr(), libc::RTLD_NOW | libc::RTLD_GLOBAL) };
if handle.is_null() {
let error = unsafe {
let err = libc::dlerror();
if err.is_null() {
"unknown dlopen error".to_string()
} else {
std::ffi::CStr::from_ptr(err).to_string_lossy().into_owned()
}
};
return Err(VmError::HostSetup(format!(
"preload runtime support library {}: {error}",
path.display()
)));
}
}
Ok(support_libs)
}
pub fn required_runtime_lib_name() -> &'static str {
#[cfg(target_os = "macos")]
{
"libkrun.dylib"
}
#[cfg(not(target_os = "macos"))]
{
"libkrun.so"
}
}
/// Compute SHA-256 hash of a file, returning hex string.
///
/// Streams the file contents directly to `shasum -a 256` via a pipe,
/// avoiding buffering the entire file in memory.
fn compute_sha256(path: &Path) -> Result<String, std::io::Error> {
use std::io::{Read, Write};
use std::process::{Command, Stdio};
let mut file = fs::File::open(path)?;
// sha256sum is standard on Linux; shasum ships with macOS/Perl.
let mut child = Command::new("sha256sum")
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
.or_else(|_| {
Command::new("shasum")
.args(["-a", "256"])
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::null())
.spawn()
})?;
// Stream file contents directly to shasum's stdin in 8KB chunks.
{
let mut stdin = child
.stdin
.take()
.ok_or_else(|| std::io::Error::other("failed to open shasum stdin"))?;
let mut buf = [0u8; 8192];
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
stdin.write_all(&buf[..n])?;
}
// stdin is dropped here, closing the pipe so shasum can finish.
}
let output = child.wait_with_output()?;
if output.status.success() {
let stdout = String::from_utf8_lossy(&output.stdout);
Ok(stdout
.split_whitespace()
.next()
.unwrap_or("unknown")
.to_string())
} else {
Ok("unknown".to_string())
}
}
fn load_symbol<T: Copy>(
library: &'static Library,
symbol: &[u8],
path: &Path,
) -> Result<T, VmError> {
let loaded = unsafe {
library.get::<T>(symbol).map_err(|e| {
VmError::HostSetup(format!(
"resolve {} from {}: {e}",
String::from_utf8_lossy(symbol).trim_end_matches('\0'),
path.display()
))
})?
};
Ok(*loaded)
}
fn load_optional_symbol<T: Copy>(library: &'static Library, symbol: &[u8]) -> Option<T> {
let loaded = unsafe { library.get::<T>(symbol).ok()? };
Some(*loaded)
}
crates/openshell-vm/src/health.rs
+201
View File
@@ -0,0 +1,201 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! gRPC health check for verifying the gateway is fully ready.
//!
//! This module provides a proper gRPC health check that verifies the gateway
//! service is not just accepting TCP connections, but is actually responding
//! to gRPC requests. This ensures we don't mark the server as ready before
//! it has fully booted.
use crate::VmError;
use openshell_core::proto::{HealthRequest, ServiceStatus, open_shell_client::OpenShellClient};
use std::path::PathBuf;
use std::time::Duration;
use tonic::transport::{Certificate, ClientTlsConfig, Endpoint, Identity};
/// Load mTLS materials from the gateway's cert directory.
fn load_mtls_materials(gateway_name: &str) -> Result<(Vec<u8>, Vec<u8>, Vec<u8>), String> {
let home = std::env::var("HOME").map_err(|_| "HOME not set")?;
let mtls_dir = PathBuf::from(home)
.join(".config/openshell/gateways")
.join(gateway_name)
.join("mtls");
let ca = std::fs::read(mtls_dir.join("ca.crt"))
.map_err(|e| format!("failed to read ca.crt: {e}"))?;
let cert = std::fs::read(mtls_dir.join("tls.crt"))
.map_err(|e| format!("failed to read tls.crt: {e}"))?;
let key = std::fs::read(mtls_dir.join("tls.key"))
.map_err(|e| format!("failed to read tls.key: {e}"))?;
Ok((ca, cert, key))
}
/// Build a tonic TLS config from mTLS materials.
fn build_tls_config(ca: Vec<u8>, cert: Vec<u8>, key: Vec<u8>) -> ClientTlsConfig {
let ca_cert = Certificate::from_pem(ca);
let identity = Identity::from_pem(cert, key);
ClientTlsConfig::new()
.ca_certificate(ca_cert)
.identity(identity)
}
/// Perform a gRPC health check against the gateway.
///
/// Returns `Ok(())` if the health check succeeds (service reports healthy),
/// or an error describing why the check failed.
async fn grpc_health_check(gateway_port: u16, gateway_name: &str) -> Result<(), String> {
// Load mTLS materials
let (ca, cert, key) = load_mtls_materials(gateway_name)?;
let tls_config = build_tls_config(ca, cert, key);
// Build the channel with TLS
let endpoint = format!("https://127.0.0.1:{gateway_port}");
let channel = Endpoint::from_shared(endpoint.clone())
.map_err(|e| format!("invalid endpoint: {e}"))?
.connect_timeout(Duration::from_secs(5))
.tls_config(tls_config)
.map_err(|e| format!("TLS config error: {e}"))?
.connect()
.await
.map_err(|e| format!("connection failed: {e}"))?;
// Create client and call health
let mut client = OpenShellClient::new(channel);
let response = client
.health(HealthRequest {})
.await
.map_err(|e| format!("health RPC failed: {e}"))?;
let health = response.into_inner();
if health.status == ServiceStatus::Healthy as i32 {
Ok(())
} else {
Err(format!("service not healthy: status={}", health.status))
}
}
/// Wait for the gateway service to be fully ready by polling the gRPC health endpoint.
///
/// This replaces the TCP-only probe with a proper gRPC health check that verifies
/// the service is actually responding to requests, not just accepting connections.
///
/// Returns `Ok(())` when the gateway is confirmed healthy, or `Err` if the health
/// check fails or times out. Falls back to TCP probe if mTLS materials aren't
/// available yet.
pub fn wait_for_gateway_ready(gateway_port: u16, gateway_name: &str) -> Result<(), VmError> {
let start = std::time::Instant::now();
let timeout = Duration::from_secs(90);
let poll_interval = Duration::from_secs(1);
eprintln!("Waiting for gateway gRPC health check...");
// Create a runtime for async health checks
let rt = match tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
{
Ok(rt) => rt,
Err(e) => {
eprintln!(" failed to create tokio runtime: {e}, falling back to TCP probe");
return wait_for_tcp_only(gateway_port, timeout, poll_interval);
}
};
loop {
// Try gRPC health check
let result = rt.block_on(async {
tokio::time::timeout(
Duration::from_secs(5),
grpc_health_check(gateway_port, gateway_name),
)
.await
});
match result {
Ok(Ok(())) => {
eprintln!("Gateway healthy [{:.1}s]", start.elapsed().as_secs_f64());
return Ok(());
}
Ok(Err(e)) => {
// gRPC call completed but failed
if start.elapsed() >= timeout {
return Err(VmError::Bootstrap(format!(
"gateway health check failed after {:.0}s: {e}",
timeout.as_secs_f64()
)));
}
}
Err(_) => {
// Timeout on the health check itself
if start.elapsed() >= timeout {
return Err(VmError::Bootstrap(format!(
"gateway health check timed out after {:.0}s",
timeout.as_secs_f64()
)));
}
}
}
std::thread::sleep(poll_interval);
}
}
/// Fallback TCP-only probe when gRPC health check can't be performed.
fn wait_for_tcp_only(
gateway_port: u16,
timeout: Duration,
poll_interval: Duration,
) -> Result<(), VmError> {
let start = std::time::Instant::now();
loop {
if host_tcp_probe(gateway_port) {
eprintln!(
"Service reachable (TCP) [{:.1}s]",
start.elapsed().as_secs_f64()
);
return Ok(());
}
if start.elapsed() >= timeout {
return Err(VmError::Bootstrap(format!(
"gateway TCP probe failed after {:.0}s",
timeout.as_secs_f64()
)));
}
std::thread::sleep(poll_interval);
}
}
/// Probe `127.0.0.1:port` from the host to verify the TCP path is working.
///
/// This is a fallback when gRPC health check isn't available.
fn host_tcp_probe(gateway_port: u16) -> bool {
use std::io::Read;
use std::net::{SocketAddr, TcpStream};
let addr: SocketAddr = ([127, 0, 0, 1], gateway_port).into();
let Ok(mut stream) = TcpStream::connect_timeout(&addr, Duration::from_secs(2)) else {
return false;
};
// A short read timeout: if the server is alive it will wait for us
// to send a TLS ClientHello, so the read will time out (= good).
// If the connection resets or closes, the server is dead.
stream
.set_read_timeout(Some(Duration::from_millis(200)))
.ok();
let mut buf = [0u8; 1];
match stream.read(&mut buf) {
Err(e)
if e.kind() == std::io::ErrorKind::WouldBlock
|| e.kind() == std::io::ErrorKind::TimedOut =>
{
true // Timeout = server alive, waiting for ClientHello.
}
_ => false, // Reset, EOF, or unexpected data = not healthy.
}
}
crates/openshell-vm/src/lib.rs
+1994
View File
File diff suppressed because it is too large Load Diff
crates/openshell-vm/src/main.rs
+241
View File
@@ -0,0 +1,241 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Standalone openshell-vm binary.
//!
//! Boots a libkrun microVM running the `OpenShell` control plane (k3s +
//! openshell-server). Each named instance gets its own rootfs extracted from
//! the embedded tarball at
//! `~/.local/share/openshell/openshell-vm/{version}/instances/<name>/rootfs`.
//!
//! # Codesigning (macOS)
//!
//! This binary must be codesigned with the `com.apple.security.hypervisor`
//! entitlement. See `entitlements.plist` in this crate.
//!
//! ```sh
//! codesign --entitlements crates/openshell-vm/entitlements.plist --force -s - target/debug/openshell-vm
//! ```
use std::io::IsTerminal;
use std::path::PathBuf;
use clap::{Parser, Subcommand, ValueHint};
const DISABLE_STATE_DISK_ENV: &str = "OPENSHELL_VM_DISABLE_STATE_DISK";
/// Boot the `OpenShell` gateway microVM.
///
/// Starts a libkrun microVM running a k3s Kubernetes cluster with the
/// `OpenShell` control plane. Use `--exec` to run a custom process instead.
#[derive(Parser)]
#[command(name = "openshell-vm", version)]
struct Cli {
#[command(subcommand)]
command: Option<GatewayCommand>,
/// Path to the rootfs directory (aarch64 Linux).
/// Overrides the default instance-based rootfs resolution.
#[arg(long, value_hint = ValueHint::DirPath)]
rootfs: Option<PathBuf>,
/// Named VM instance.
///
/// When used alone, the rootfs resolves to
/// `~/.local/share/openshell/openshell-vm/{version}/instances/<name>/rootfs`
/// and is extracted from the embedded tarball on first use.
/// When combined with `--rootfs`, only provides the instance identity
/// (for exec, gateway name, etc.) while the rootfs comes from the
/// explicit path.
#[arg(long, default_value = "default")]
name: String,
/// Executable path inside the VM. When set, runs this instead of
/// the default k3s server.
#[arg(long)]
exec: Option<String>,
/// Arguments to the executable (requires `--exec`).
#[arg(long, num_args = 1..)]
args: Vec<String>,
/// Environment variables in `KEY=VALUE` form (requires `--exec`).
#[arg(long, num_args = 1..)]
env: Vec<String>,
/// Working directory inside the VM.
#[arg(long, default_value = "/")]
workdir: String,
/// Port mappings (`host_port:guest_port`).
#[arg(long, short, num_args = 1..)]
port: Vec<String>,
/// Number of virtual CPUs (default: 4 for openshell-vm, 2 for --exec).
#[arg(long)]
vcpus: Option<u8>,
/// RAM in MiB (default: 8192 for openshell-vm, 2048 for --exec).
#[arg(long)]
mem: Option<u32>,
/// libkrun log level (0=Off .. 5=Trace).
#[arg(long, default_value_t = 1)]
krun_log_level: u32,
/// Networking backend: "gvproxy" (default), "tsi", or "none".
#[arg(long, default_value = "gvproxy")]
net: String,
/// Wipe all runtime state (containerd, kubelet, k3s) before booting.
/// Use this to recover from a corrupted state after a crash or
/// unclean shutdown.
#[arg(long)]
reset: bool,
}
#[derive(Subcommand)]
enum GatewayCommand {
/// Ensure the target rootfs exists, extracting the embedded rootfs if needed.
PrepareRootfs {
/// Recreate the target rootfs even if it already exists.
#[arg(long)]
force: bool,
},
/// Execute a command inside a running openshell-vm VM.
Exec {
/// Working directory inside the VM.
#[arg(long)]
workdir: Option<String>,
/// Environment variables in `KEY=VALUE` form.
#[arg(long, num_args = 1..)]
env: Vec<String>,
/// Command and arguments to run inside the VM.
#[arg(trailing_var_arg = true)]
command: Vec<String>,
},
}
fn main() {
tracing_subscriber::fmt::init();
let cli = Cli::parse();
let code = match run(cli) {
Ok(code) => code,
Err(e) => {
eprintln!("Error: {e}");
1
}
};
if code != 0 {
std::process::exit(code);
}
}
fn run(cli: Cli) -> Result<i32, Box<dyn std::error::Error>> {
if let Some(GatewayCommand::PrepareRootfs { force }) = &cli.command {
let rootfs = openshell_vm::prepare_rootfs(cli.rootfs.clone(), &cli.name, *force)?;
println!("{}", rootfs.display());
return Ok(0);
}
if let Some(GatewayCommand::Exec {
workdir,
env,
mut command,
}) = cli.command
{
let effective_tty = std::io::stdin().is_terminal();
if command.is_empty() {
if effective_tty {
command.push("sh".to_string());
} else {
return Err("openshell-vm exec requires a command when stdin is not a TTY".into());
}
}
return Ok(openshell_vm::exec_running_vm(
openshell_vm::VmExecOptions {
rootfs: Some(
cli.rootfs
.unwrap_or(openshell_vm::named_rootfs_dir(&cli.name)?),
),
command,
workdir,
env,
tty: effective_tty,
},
)?);
}
let net_backend = match cli.net.as_str() {
"tsi" => openshell_vm::NetBackend::Tsi,
"none" => openshell_vm::NetBackend::None,
"gvproxy" => openshell_vm::NetBackend::Gvproxy {
binary: openshell_vm::default_runtime_gvproxy_path(),
},
other => {
return Err(
format!("unknown --net backend: {other} (expected: gvproxy, tsi, none)").into(),
);
}
};
let rootfs = cli
.rootfs
.map_or_else(|| openshell_vm::ensure_named_rootfs(&cli.name), Ok)?;
let gateway_name = openshell_vm::gateway_name(&cli.name)?;
let mut config = if let Some(exec_path) = cli.exec {
openshell_vm::VmConfig {
rootfs,
vcpus: cli.vcpus.unwrap_or(2),
mem_mib: cli.mem.unwrap_or(2048),
exec_path,
args: cli.args,
env: cli.env,
workdir: cli.workdir,
port_map: cli.port,
vsock_ports: vec![],
log_level: cli.krun_log_level,
console_output: None,
net: net_backend,
reset: cli.reset,
gateway_name,
state_disk: None,
}
} else {
let mut c = openshell_vm::VmConfig::gateway(rootfs);
if !cli.port.is_empty() {
c.port_map = cli.port;
}
if let Some(v) = cli.vcpus {
c.vcpus = v;
}
if let Some(m) = cli.mem {
c.mem_mib = m;
}
c.net = net_backend;
c.reset = cli.reset;
c.gateway_name = gateway_name;
if state_disk_disabled() {
c.state_disk = None;
}
c
};
config.log_level = cli.krun_log_level;
Ok(openshell_vm::launch(&config)?)
}
fn state_disk_disabled() -> bool {
matches!(
std::env::var(DISABLE_STATE_DISK_ENV).ok().as_deref(),
Some("1" | "true" | "TRUE" | "yes" | "YES")
)
}