feat(firecracker): prototype host-supervised isolation

Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com>
This commit is contained in:
Drew Newberry
2026-08-25 16:23:40 +00:00
parent 531ffd5b64
commit b4bd85eb6f
15 changed files with 2420 additions and 0 deletions
+1
View File
@@ -47,6 +47,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files
| `crates/openshell-driver-docker/` | Docker compute driver | In-process `ComputeDriver` backend for local Docker sandbox containers |
| `crates/openshell-driver-podman/` | Podman compute driver | In-process `ComputeDriver` backend for local Podman sandbox containers |
| `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` subprocess (embeds its own rootfs + runtime) |
| `crates/openshell-driver-firecracker/` | Firecracker isolation driver | Experimental host-side RFC 0012 backend with a private guest process-supervisor leaf mode |
| `crates/openshell-prover/` | Policy prover | Policy verification and proof generation |
| `crates/openshell-server-macros/` | Server macros | Compile-time helpers for gateway RPC authorization |
| `crates/openshell-supervisor-middleware/` | Middleware runtime | Generic middleware registry, remote service integration, and chain execution |
Generated
+18
View File
@@ -3920,6 +3920,24 @@ dependencies = [
"url",
]
[[package]]
name = "openshell-driver-firecracker"
version = "0.0.0"
dependencies = [
"async-trait",
"clap",
"libc",
"nix 0.29.0",
"openshell-core",
"openshell-isolation",
"openshell-supervisor-process",
"serde",
"serde_json",
"tempfile",
"thiserror 2.0.18",
"tokio",
]
[[package]]
name = "openshell-driver-kubernetes"
version = "0.0.0"
+1
View File
@@ -158,6 +158,7 @@ Runtime-specific implementation notes belong in the driver crate README:
- `crates/openshell-driver-podman/README.md`
- `crates/openshell-driver-kubernetes/README.md`
- `crates/openshell-driver-vm/README.md`
- `crates/openshell-driver-firecracker/README.md`
The combined VM topology runs `openshell-sandbox` as guest PID 1. libkrun
executes the driver-owned guest bootstrap as PID 1, and the bootstrap preserves
+8
View File
@@ -70,6 +70,14 @@ by default. The co-located backend requires the
supervisor to own the execution environment's PID namespace so boundary
teardown can terminate every remaining workload process.
The experimental Firecracker backend exercises a delegated placement. The
logical supervisor stays on the host and sends the admitted policy and workload
spec over an authenticated, backend-private virtio-vsock channel only after the
RFC lifecycle reaches `start_agent`. A private guest mode of the Firecracker
driver invokes the existing process-supervisor implementation as the in-VM
leaf. The prototype attaches no guest NIC, so network access is blocked by the
VM topology while mediated egress remains unimplemented.
For proxy-mode boundaries, the co-located backend verifies its default-deny
kernel egress ceiling before exposing any workload execution surface and then
rechecks it every 250 milliseconds. Each check has a two-second deadline.
@@ -0,0 +1,39 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
[package]
name = "openshell-driver-firecracker"
description = "Experimental Firecracker isolation driver for OpenShell"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
repository.workspace = true
[lib]
name = "openshell_driver_firecracker"
path = "src/lib.rs"
[[bin]]
name = "openshell-driver-firecracker"
path = "src/main.rs"
[dependencies]
openshell-core = { path = "../openshell-core", default-features = false }
openshell-isolation = { path = "../openshell-isolation" }
openshell-supervisor-process = { path = "../openshell-supervisor-process" }
async-trait = "0.1"
clap = { workspace = true }
libc = "0.2"
nix = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true }
[dev-dependencies]
tempfile = "3"
[lints]
workspace = true
@@ -0,0 +1,37 @@
# OpenShell Firecracker driver
`openshell-driver-firecracker` is an experimental host-side implementation of
the RFC 0012 Isolation Backend contract. It deliberately lives outside the
existing libkrun VM driver so the two runtimes can evolve independently.
The main OpenShell supervisor remains on the host. A private mode of the same
driver binary listens on the guest's virtio-vsock device and invokes the current
`openshell-supervisor-process` implementation only after the host advances the
boundary through `attach -> confirm -> start_agent`. The admitted policy crosses
that authenticated channel with the workload spec. The private protocol is not
a second public supervisor or agent API.
The prototype has no virtual NIC. This makes the network ceiling structurally
fail closed without TAP devices, nftables, `CAP_NET_ADMIN`, or `sudo`. Host
requirements are a Linux Firecracker binary and read/write access to `/dev/kvm`.
Current scope:
- boots an existing ext4 guest image with Firecracker;
- authenticates host-to-guest control over virtio-vsock;
- implements the RFC lifecycle and agent wait/signal operations;
- delegates guest process enforcement to the existing process supervisor leaf;
- provides an unprivileged KVM end-to-end smoke runner.
Exec, PTY, port forwarding, mediated guest egress, and per-connection binary
identity are intentionally deferred. The contract surfaces fail closed for
those operations. Code that might later become a shared VM helper is duplicated
here until a second consumer establishes a small, stable abstraction.
Run the smoke test with:
```shell
mise run e2e:firecracker
```
See `e2e/firecracker/README.md` for fixture overrides.
@@ -0,0 +1,493 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Host-side RFC 0012 backend for an already-provisioned Firecracker VM.
use std::fmt;
use std::path::PathBuf;
use std::sync::Arc;
use std::sync::atomic::{AtomicU64, Ordering};
use std::time::Duration;
use async_trait::async_trait;
use openshell_isolation::AgentSpec;
use openshell_isolation::contract::{
BackendError, BoundBoundary, BoundaryDuplexStream, BoundaryExec, BoundaryExitStatus,
BoundaryPortForward, BoundaryProcess, BoundarySignal, ExecSession, ExecSpec, INTERFACE_VERSION,
IsolationBackend, LoopbackTarget, MediatedConnection, NetworkMediationSource, ReadyBoundary,
RunningBoundary, SandboxContext, VerifiedTopologyDescriptor,
};
use serde::{Deserialize, Serialize};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::UnixStream;
use crate::protocol::{
AgentSpecWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, Response, ResponseEnvelope,
SandboxPolicyWire, SignalWire, decode_frame, encode_frame,
};
pub const BACKEND_NAME: &str = "firecracker";
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
const MIN_BOOTSTRAP_TOKEN_BYTES: usize = 32;
const MAX_VSOCK_ACK_BYTES: usize = 64;
/// Backend-private provisioned topology payload.
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct FirecrackerTopology {
pub boundary_id: String,
pub vsock_uds_path: PathBuf,
pub control_port: u32,
pub bootstrap_token: String,
}
impl fmt::Debug for FirecrackerTopology {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("FirecrackerTopology")
.field("boundary_id", &self.boundary_id)
.field("vsock_uds_path", &self.vsock_uds_path)
.field("control_port", &self.control_port)
.field("bootstrap_token", &"<redacted>")
.finish()
}
}
impl FirecrackerTopology {
pub fn encode(&self) -> Result<Vec<u8>, BackendError> {
serde_json::to_vec(self)
.map_err(|error| BackendError::Descriptor(format!("encode topology: {error}")))
}
}
/// Host-side Firecracker implementation registered with the supervisor.
#[derive(Debug, Default)]
pub struct FirecrackerHostBackend;
#[async_trait]
impl IsolationBackend for FirecrackerHostBackend {
fn backend_name(&self) -> &str {
BACKEND_NAME
}
fn version(&self) -> u32 {
INTERFACE_VERSION
}
async fn attach(
&self,
descriptor: VerifiedTopologyDescriptor,
sandbox: SandboxContext,
) -> Result<Box<dyn BoundBoundary>, BackendError> {
let topology: FirecrackerTopology = serde_json::from_slice(descriptor.payload())
.map_err(|error| BackendError::Descriptor(format!("decode topology: {error}")))?;
validate_topology(&topology, &sandbox)?;
let client = Arc::new(GuestClient::new(topology));
expect_response(client.call(Request::Attach).await?, "attached")?;
Ok(Box::new(FirecrackerBound {
client,
agent: sandbox.agent,
policy: sandbox.policy,
sandbox_id: sandbox.sandbox_id,
mediation: Arc::new(NoGuestNetwork),
}))
}
}
fn validate_topology(
topology: &FirecrackerTopology,
sandbox: &SandboxContext,
) -> Result<(), BackendError> {
if topology.boundary_id != sandbox.sandbox_id {
return Err(BackendError::Descriptor(format!(
"Firecracker boundary {:?} does not match sandbox {:?}",
topology.boundary_id, sandbox.sandbox_id
)));
}
if topology.bootstrap_token.len() < MIN_BOOTSTRAP_TOKEN_BYTES {
return Err(BackendError::Descriptor(format!(
"Firecracker bootstrap token must be at least {MIN_BOOTSTRAP_TOKEN_BYTES} bytes"
)));
}
if !topology.vsock_uds_path.is_absolute() {
return Err(BackendError::Descriptor(
"Firecracker vsock UDS path must be absolute".to_string(),
));
}
if topology.control_port == 0 {
return Err(BackendError::Descriptor(
"Firecracker guest control port must be nonzero".to_string(),
));
}
Ok(())
}
struct FirecrackerBound {
client: Arc<GuestClient>,
agent: AgentSpec,
policy: openshell_core::policy::SandboxPolicy,
sandbox_id: String,
mediation: Arc<NoGuestNetwork>,
}
#[async_trait]
impl BoundBoundary for FirecrackerBound {
fn network_mediation_source(&self) -> Arc<dyn NetworkMediationSource> {
self.mediation.clone()
}
async fn confirm(self: Box<Self>) -> Result<Box<dyn ReadyBoundary>, BackendError> {
expect_response(self.client.call(Request::Confirm).await?, "confirmed")?;
Ok(Box::new(FirecrackerReady {
client: self.client,
agent: self.agent,
policy: self.policy,
sandbox_id: self.sandbox_id,
}))
}
}
struct FirecrackerReady {
client: Arc<GuestClient>,
agent: AgentSpec,
policy: openshell_core::policy::SandboxPolicy,
sandbox_id: String,
}
#[async_trait]
impl ReadyBoundary for FirecrackerReady {
async fn start_agent(self: Box<Self>) -> Result<Box<dyn RunningBoundary>, BackendError> {
let response = self
.client
.call(Request::StartAgent {
sandbox_id: self.sandbox_id,
spec: AgentSpecWire::from(self.agent),
policy: Box::new(SandboxPolicyWire::from(self.policy)),
})
.await?;
let Response::Started { process_id } = response else {
return Err(unexpected_response("started", &response));
};
let process = Arc::new(FirecrackerProcess {
client: self.client,
process_id,
});
Ok(Box::new(FirecrackerRunning {
process,
exec: Arc::new(UnsupportedExec),
port_forward: Arc::new(UnsupportedPortForward),
}))
}
}
struct FirecrackerRunning {
process: Arc<FirecrackerProcess>,
exec: Arc<UnsupportedExec>,
port_forward: Arc<UnsupportedPortForward>,
}
impl RunningBoundary for FirecrackerRunning {
fn agent(&self) -> Arc<dyn BoundaryProcess> {
self.process.clone()
}
fn exec(&self) -> Arc<dyn BoundaryExec> {
self.exec.clone()
}
fn port_forward(&self) -> Arc<dyn BoundaryPortForward> {
self.port_forward.clone()
}
}
struct FirecrackerProcess {
client: Arc<GuestClient>,
process_id: String,
}
#[async_trait]
impl BoundaryProcess for FirecrackerProcess {
async fn wait(&self) -> Result<BoundaryExitStatus, BackendError> {
let response = self
.client
.call_wait(Request::Wait {
process_id: self.process_id.clone(),
})
.await?;
let Response::Exited { status } = response else {
return Err(unexpected_response("exited", &response));
};
Ok(status.into())
}
async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> {
let response = self
.client
.call(Request::Signal {
process_id: self.process_id.clone(),
signal: SignalWire::from(signal),
})
.await?;
expect_response(response, "signaled")
}
async fn terminate(&self) -> Result<(), BackendError> {
let response = self
.client
.call(Request::Terminate {
process_id: self.process_id.clone(),
})
.await?;
expect_response(response, "terminated")
}
}
struct UnsupportedExec;
#[async_trait]
impl BoundaryExec for UnsupportedExec {
async fn exec(&self, _spec: ExecSpec) -> Result<ExecSession, BackendError> {
Err(BackendError::Unavailable(
"Firecracker exec and PTY transport are not implemented in this prototype".to_string(),
))
}
}
struct UnsupportedPortForward;
#[async_trait]
impl BoundaryPortForward for UnsupportedPortForward {
async fn connect(&self, _target: LoopbackTarget) -> Result<BoundaryDuplexStream, BackendError> {
Err(BackendError::Unavailable(
"Firecracker loopback forwarding is not implemented in this prototype".to_string(),
))
}
}
/// The prototype provisions no guest NIC. There can be no workload egress to
/// accept, and attempting to consume the source fails the boundary closed.
struct NoGuestNetwork;
#[async_trait]
impl NetworkMediationSource for NoGuestNetwork {
async fn accept(&self) -> Result<MediatedConnection, BackendError> {
Err(BackendError::Unavailable(
"Firecracker guest has no NIC; mediated egress is unavailable".to_string(),
))
}
}
struct GuestClient {
topology: FirecrackerTopology,
next_request_id: AtomicU64,
}
impl GuestClient {
fn new(topology: FirecrackerTopology) -> Self {
Self {
topology,
next_request_id: AtomicU64::new(1),
}
}
async fn call(&self, request: Request) -> Result<Response, BackendError> {
tokio::time::timeout(REQUEST_TIMEOUT, self.exchange(request))
.await
.map_err(|_| BackendError::Unavailable("guest control request timed out".to_string()))?
}
async fn call_wait(&self, request: Request) -> Result<Response, BackendError> {
self.exchange(request).await
}
async fn exchange(&self, request: Request) -> Result<Response, BackendError> {
let request_id = self.next_request_id.fetch_add(1, Ordering::Relaxed);
let envelope = RequestEnvelope {
request_id,
boundary_id: self.topology.boundary_id.clone(),
bootstrap_token: self.topology.bootstrap_token.clone(),
request,
};
let mut stream = self.connect_vsock().await?;
let frame = encode_frame(&envelope)
.map_err(|error| BackendError::Process(format!("encode control request: {error}")))?;
stream.write_all(&frame).await.map_err(|error| {
BackendError::Unavailable(format!("write guest control request: {error}"))
})?;
let mut header = [0_u8; 4];
stream.read_exact(&mut header).await.map_err(|error| {
BackendError::Unavailable(format!("read guest control response header: {error}"))
})?;
let declared = u32::from_be_bytes(header) as usize;
if declared > MAX_CONTROL_FRAME_BYTES {
return Err(BackendError::Process(format!(
"guest control response is too large: {declared} bytes"
)));
}
let mut frame = Vec::with_capacity(4 + declared);
frame.extend_from_slice(&header);
frame.resize(4 + declared, 0);
stream.read_exact(&mut frame[4..]).await.map_err(|error| {
BackendError::Unavailable(format!("read guest control response: {error}"))
})?;
let response: ResponseEnvelope = decode_frame(&frame)
.map_err(|error| BackendError::Process(format!("decode control response: {error}")))?;
if response.request_id != request_id {
return Err(BackendError::Process(format!(
"guest response ID {} did not match request ID {request_id}",
response.request_id
)));
}
match response.response {
Response::Error { kind, message } => Err(guest_error(&kind, message)),
response => Ok(response),
}
}
async fn connect_vsock(&self) -> Result<UnixStream, BackendError> {
let mut stream = UnixStream::connect(&self.topology.vsock_uds_path)
.await
.map_err(|error| {
BackendError::Unavailable(format!("connect to Firecracker vsock UDS: {error}"))
})?;
stream
.write_all(format!("CONNECT {}\n", self.topology.control_port).as_bytes())
.await
.map_err(|error| {
BackendError::Unavailable(format!("write Firecracker vsock handshake: {error}"))
})?;
let acknowledgment = read_vsock_acknowledgment(&mut stream).await?;
validate_vsock_acknowledgment(&acknowledgment)?;
Ok(stream)
}
}
async fn read_vsock_acknowledgment(stream: &mut UnixStream) -> Result<Vec<u8>, BackendError> {
let mut acknowledgment = Vec::with_capacity(24);
loop {
if acknowledgment.len() == MAX_VSOCK_ACK_BYTES {
return Err(BackendError::Process(
"Firecracker vsock acknowledgment is too long".to_string(),
));
}
let byte = stream.read_u8().await.map_err(|error| {
BackendError::Unavailable(format!("read Firecracker vsock acknowledgment: {error}"))
})?;
acknowledgment.push(byte);
if byte == b'\n' {
return Ok(acknowledgment);
}
}
}
fn validate_vsock_acknowledgment(acknowledgment: &[u8]) -> Result<(), BackendError> {
let acknowledgment = std::str::from_utf8(acknowledgment).map_err(|_| {
BackendError::Process("Firecracker vsock acknowledgment is not UTF-8".to_string())
})?;
let mut fields = acknowledgment
.trim_end_matches('\n')
.split_ascii_whitespace();
let status = fields.next();
let assigned_port = fields.next().and_then(|port| port.parse::<u32>().ok());
if status != Some("OK") || assigned_port.is_none() || fields.next().is_some() {
return Err(BackendError::Process(format!(
"invalid Firecracker vsock acknowledgment: {acknowledgment:?}"
)));
}
Ok(())
}
fn expect_response(response: Response, expected: &str) -> Result<(), BackendError> {
let matches = matches!(
(&response, expected),
(Response::Attached, "attached")
| (Response::Confirmed, "confirmed")
| (Response::Signaled, "signaled")
| (Response::Terminated, "terminated")
);
if matches {
Ok(())
} else {
Err(unexpected_response(expected, &response))
}
}
fn unexpected_response(expected: &str, response: &Response) -> BackendError {
BackendError::Process(format!(
"expected guest response {expected:?}, received {response:?}"
))
}
fn guest_error(kind: &str, message: String) -> BackendError {
let message = format!("Firecracker guest process leaf: {message}");
match kind {
"invalid" => BackendError::Descriptor(message),
"denied" => BackendError::Denied(message),
"unavailable" => BackendError::Unavailable(message),
"terminated" => BackendError::Terminated(message),
_ => BackendError::Process(message),
}
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use openshell_core::policy::{
FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy,
};
use super::*;
fn sandbox() -> SandboxContext {
SandboxContext {
sandbox_id: "sandbox-1".to_string(),
policy: SandboxPolicy {
version: 1,
filesystem: FilesystemPolicy::default(),
network: NetworkPolicy::default(),
landlock: LandlockPolicy::default(),
process: ProcessPolicy::default(),
},
agent: AgentSpec {
program: "/bin/true".to_string(),
args: Vec::new(),
workdir: Some("/sandbox".to_string()),
timeout_secs: 5,
interactive: false,
},
}
}
#[test]
fn topology_debug_redacts_token() {
let topology = FirecrackerTopology {
boundary_id: "sandbox-1".to_string(),
vsock_uds_path: PathBuf::from("/tmp/vsock.sock"),
control_port: 5500,
bootstrap_token: "never-log-this-never-log-this".to_string(),
};
let debug = format!("{topology:?}");
assert!(debug.contains("<redacted>"));
assert!(!debug.contains("never-log-this"));
}
#[test]
fn topology_must_match_sandbox() {
let topology = FirecrackerTopology {
boundary_id: "other".to_string(),
vsock_uds_path: PathBuf::from("/tmp/vsock.sock"),
control_port: 5500,
bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(),
};
assert!(matches!(
validate_topology(&topology, &sandbox()),
Err(BackendError::Descriptor(_))
));
}
#[test]
fn validates_firecracker_acknowledgment() {
assert!(validate_vsock_acknowledgment(b"OK 1234\n").is_ok());
assert!(validate_vsock_acknowledgment(b"ERR\n").is_err());
}
}
@@ -0,0 +1,601 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Private guest mode for the Firecracker driver.
//!
//! This is transport and lifecycle glue, not another supervisor model. When
//! the host authorizes `start_agent`, it invokes the existing
//! `openshell-supervisor-process` implementation inside the VM.
#![allow(unsafe_code)]
use std::ffi::CString;
use std::fs::File;
use std::io::{self, Read, Write};
use std::mem::size_of;
use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
use std::path::Path;
use std::sync::atomic::AtomicU32;
use std::sync::{Arc, Condvar, Mutex};
use std::time::Duration;
use openshell_core::proposals::AgentProposals;
use openshell_core::provider_credentials::ProviderCredentialState;
use openshell_supervisor_process::boundary_io::BoundaryRuntimeState;
use openshell_supervisor_process::process::{
ProcessEnforcementMode, ProcessStatus, ResolvedProcessIdentity,
};
use openshell_supervisor_process::run::{AgentSignaler, spawn_workload};
use serde::{Deserialize, Serialize};
use crate::protocol::{
AgentSpecWire, ExitStatusWire, Request, RequestEnvelope, Response, ResponseEnvelope,
SandboxPolicyWire, SignalWire, read_frame, write_frame,
};
pub const DEFAULT_GUEST_CONFIG_PATH: &str = "/etc/openshell/firecracker.json";
const DEFAULT_CONTROL_PORT: u32 = 5500;
const DEFAULT_AGENT_UID: u32 = 10_001;
const DEFAULT_AGENT_GID: u32 = 10_001;
const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30);
/// Driver-private configuration injected into the guest image at provision time.
#[derive(Clone, Serialize, Deserialize)]
pub struct GuestConfig {
pub boundary_id: String,
pub bootstrap_token: String,
#[serde(default = "default_control_port")]
pub control_port: u32,
#[serde(default = "default_agent_uid")]
pub agent_uid: u32,
#[serde(default = "default_agent_gid")]
pub agent_gid: u32,
}
impl std::fmt::Debug for GuestConfig {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("GuestConfig")
.field("boundary_id", &self.boundary_id)
.field("bootstrap_token", &"<redacted>")
.field("control_port", &self.control_port)
.field("agent_uid", &self.agent_uid)
.field("agent_gid", &self.agent_gid)
.finish()
}
}
const fn default_control_port() -> u32 {
DEFAULT_CONTROL_PORT
}
const fn default_agent_uid() -> u32 {
DEFAULT_AGENT_UID
}
const fn default_agent_gid() -> u32 {
DEFAULT_AGENT_GID
}
pub fn run_guest(config_path: &Path) -> Result<(), String> {
if std::process::id() == 1 {
prepare_pid1_filesystems()?;
}
let bytes = std::fs::read(config_path)
.map_err(|error| format!("read guest config {}: {error}", config_path.display()))?;
let config: GuestConfig = serde_json::from_slice(&bytes)
.map_err(|error| format!("decode guest config {}: {error}", config_path.display()))?;
validate_config(&config)?;
let process_runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|error| format!("create guest process runtime: {error}"))?;
let runtime = Arc::new(GuestRuntime::new(
config.clone(),
process_runtime.handle().clone(),
));
serve(config.control_port, runtime)
}
fn validate_config(config: &GuestConfig) -> Result<(), String> {
if config.boundary_id.is_empty() {
return Err("guest boundary ID must not be empty".to_string());
}
if config.bootstrap_token.len() < 32 {
return Err("guest bootstrap token must contain at least 32 bytes".to_string());
}
if config.control_port == 0 {
return Err("guest control port must be nonzero".to_string());
}
if config.agent_uid == 0 || config.agent_gid == 0 {
return Err("guest agent UID and GID must be nonzero".to_string());
}
Ok(())
}
fn prepare_pid1_filesystems() -> Result<(), String> {
for path in ["/proc", "/sys", "/dev", "/run", "/tmp", "/sandbox"] {
std::fs::create_dir_all(path).map_err(|error| format!("create {path}: {error}"))?;
}
mount_if_needed("proc", "/proc", "proc")?;
mount_if_needed("sysfs", "/sys", "sysfs")?;
mount_if_needed("devtmpfs", "/dev", "devtmpfs")?;
std::fs::create_dir_all("/dev/pts").map_err(|error| format!("create /dev/pts: {error}"))?;
mount_if_needed("devpts", "/dev/pts", "devpts")?;
Ok(())
}
fn mount_if_needed(source: &str, target: &str, file_system: &str) -> Result<(), String> {
let source = CString::new(source).map_err(|error| error.to_string())?;
let target_c = CString::new(target).map_err(|error| error.to_string())?;
let file_system = CString::new(file_system).map_err(|error| error.to_string())?;
let result = unsafe {
libc::mount(
source.as_ptr(),
target_c.as_ptr(),
file_system.as_ptr(),
0,
std::ptr::null(),
)
};
if result == 0 {
return Ok(());
}
let error = io::Error::last_os_error();
if error.raw_os_error() == Some(libc::EBUSY) {
Ok(())
} else {
Err(format!("mount {file_system:?} on {target}: {error}"))
}
}
fn serve(port: u32, runtime: Arc<GuestRuntime>) -> Result<(), String> {
let listener = VsockListener::bind(port)
.map_err(|error| format!("bind guest control vsock port {port}: {error}"))?;
eprintln!("Firecracker process supervisor leaf listening on vsock port {port}");
loop {
match listener.accept() {
Ok(stream) => {
let runtime = runtime.clone();
std::thread::spawn(move || {
if let Err(error) = serve_one(stream, &runtime) {
eprintln!("Firecracker guest control request failed: {error}");
}
});
}
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
Err(error) => return Err(format!("accept guest control connection: {error}")),
}
}
}
fn serve_one(mut stream: VsockStream, runtime: &GuestRuntime) -> Result<(), String> {
stream
.set_timeout(CONTROL_IO_TIMEOUT)
.map_err(|error| format!("set control timeout: {error}"))?;
let request: RequestEnvelope =
read_frame(&mut stream).map_err(|error| format!("read control frame: {error}"))?;
let response = ResponseEnvelope {
request_id: request.request_id,
response: runtime.dispatch(request),
};
write_frame(&mut stream, &response).map_err(|error| format!("write control frame: {error}"))
}
struct GuestRuntime {
config: GuestConfig,
process_runtime: tokio::runtime::Handle,
state: Mutex<RuntimeState>,
}
enum RuntimeState {
AwaitingAttach,
Bound,
Ready,
Running(Arc<ManagedProcess>),
}
impl GuestRuntime {
fn new(config: GuestConfig, process_runtime: tokio::runtime::Handle) -> Self {
Self {
config,
process_runtime,
state: Mutex::new(RuntimeState::AwaitingAttach),
}
}
fn dispatch(&self, envelope: RequestEnvelope) -> Response {
if !constant_time_eq(
envelope.boundary_id.as_bytes(),
self.config.boundary_id.as_bytes(),
) || !constant_time_eq(
envelope.bootstrap_token.as_bytes(),
self.config.bootstrap_token.as_bytes(),
) {
return guest_error("denied", "control authentication failed");
}
match envelope.request {
Request::Attach => self.attach(),
Request::Confirm => self.confirm(),
Request::StartAgent {
sandbox_id,
spec,
policy,
} => self.start_agent(sandbox_id, spec, *policy),
Request::Wait { process_id } => self.wait(&process_id),
Request::Signal { process_id, signal } => self.signal(&process_id, signal),
Request::Terminate { process_id } => self.terminate(&process_id),
}
}
fn attach(&self) -> Response {
let mut state = lock(&self.state);
match *state {
RuntimeState::AwaitingAttach => {
*state = RuntimeState::Bound;
Response::Attached
}
RuntimeState::Bound => Response::Attached,
_ => guest_error("invalid", "boundary has already advanced past attach"),
}
}
fn confirm(&self) -> Response {
let mut state = lock(&self.state);
match *state {
RuntimeState::Bound => {
*state = RuntimeState::Ready;
Response::Confirmed
}
RuntimeState::Ready => Response::Confirmed,
RuntimeState::AwaitingAttach => {
guest_error("invalid", "boundary must be attached before confirm")
}
RuntimeState::Running(_) => {
guest_error("invalid", "boundary has already started its agent")
}
}
}
fn start_agent(
&self,
sandbox_id: String,
spec: AgentSpecWire,
policy: SandboxPolicyWire,
) -> Response {
let mut state = lock(&self.state);
if !matches!(*state, RuntimeState::Ready) {
return guest_error("invalid", "boundary must be confirmed before start_agent");
}
let process = match ManagedProcess::spawn(
&self.process_runtime,
sandbox_id,
spec,
policy.into(),
ResolvedProcessIdentity::new(Some(self.config.agent_uid), Some(self.config.agent_gid)),
) {
Ok(process) => Arc::new(process),
Err(error) => return guest_error("failed", error),
};
let process_id = process.process_id();
*state = RuntimeState::Running(process);
Response::Started { process_id }
}
fn wait(&self, process_id: &str) -> Response {
let process = match self.running_process(process_id) {
Ok(process) => process,
Err(response) => return response,
};
match process.wait() {
Ok(status) => Response::Exited { status },
Err(error) => guest_error("failed", error),
}
}
fn signal(&self, process_id: &str, signal: SignalWire) -> Response {
let process = match self.running_process(process_id) {
Ok(process) => process,
Err(response) => return response,
};
match process.signal(signal) {
Ok(()) => Response::Signaled,
Err(error) => guest_error("terminated", error),
}
}
fn terminate(&self, process_id: &str) -> Response {
let process = match self.running_process(process_id) {
Ok(process) => process,
Err(response) => return response,
};
match process.signal(SignalWire::Kill) {
Ok(()) => Response::Terminated,
Err(_) if process.has_exited() => Response::Terminated,
Err(error) => guest_error("failed", error),
}
}
fn running_process(&self, process_id: &str) -> Result<Arc<ManagedProcess>, Response> {
let state = lock(&self.state);
let RuntimeState::Running(process) = &*state else {
return Err(guest_error("invalid", "agent process has not been started"));
};
if process.process_id() != process_id {
return Err(guest_error("invalid", "unknown process ID"));
}
Ok(process.clone())
}
}
type ProcessExit = Result<ExitStatusWire, String>;
type SharedProcessExit = Arc<(Mutex<Option<ProcessExit>>, Condvar)>;
struct ManagedProcess {
pid: i32,
signaler: AgentSignaler,
exit: SharedProcessExit,
}
impl ManagedProcess {
fn spawn(
runtime: &tokio::runtime::Handle,
sandbox_id: String,
spec: AgentSpecWire,
policy: openshell_core::policy::SandboxPolicy,
resolved_identity: ResolvedProcessIdentity,
) -> Result<Self, String> {
if spec.program.is_empty() {
return Err("agent program must not be empty".to_string());
}
let boundary_runtime = BoundaryRuntimeState::new();
let mut spawned = runtime
.block_on(spawn_workload(
&spec.program,
&spec.args,
spec.workdir.as_deref(),
spec.timeout_secs,
spec.interactive,
Some(&sandbox_id),
None,
None,
false,
&policy,
resolved_identity,
ProcessEnforcementMode::Full,
Arc::new(AtomicU32::new(0)),
None,
ProviderCredentialState::from_child_env_snapshot(
0,
std::collections::HashMap::new(),
),
std::collections::HashMap::new(),
None,
AgentProposals::default(),
None,
None,
None,
Some(boundary_runtime),
))
.map_err(|error| format!("start process supervisor leaf: {error}"))?;
let pid = i32::try_from(spawned.pid())
.map_err(|_| "process supervisor PID does not fit i32".to_string())?;
let signaler = spawned.signaler();
let exit = Arc::new((Mutex::new(None), Condvar::new()));
let reaper_exit = exit.clone();
runtime.spawn(async move {
let result = spawned
.wait()
.await
.map(process_status)
.map_err(|error| format!("wait for process supervisor leaf: {error}"));
let (state, changed) = &*reaper_exit;
*lock(state) = Some(result);
changed.notify_all();
});
Ok(Self {
pid,
signaler,
exit,
})
}
fn process_id(&self) -> String {
self.pid.to_string()
}
fn wait(&self) -> ProcessExit {
let (state, changed) = &*self.exit;
let mut exit = lock(state);
while exit.is_none() {
exit = changed
.wait(exit)
.unwrap_or_else(std::sync::PoisonError::into_inner);
}
exit.as_ref().expect("exit checked above").clone()
}
fn signal(&self, signal: SignalWire) -> Result<(), String> {
if self.has_exited() {
return Err("agent process has already exited".to_string());
}
let result = match signal {
SignalWire::Term => self.signaler.term(),
SignalWire::Kill => self.signaler.kill(),
SignalWire::Int => self.signaler.interrupt(),
SignalWire::Hup => self.signaler.hangup(),
};
result.map_err(|error| format!("signal process supervisor group: {error}"))
}
fn has_exited(&self) -> bool {
let (state, _) = &*self.exit;
lock(state).is_some()
}
}
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
mutex
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn process_status(status: ProcessStatus) -> ExitStatusWire {
status.signal().map_or_else(
|| ExitStatusWire::Exited(status.code()),
ExitStatusWire::Signaled,
)
}
fn guest_error(kind: &str, message: impl Into<String>) -> Response {
Response::Error {
kind: kind.to_string(),
message: message.into(),
}
}
fn constant_time_eq(left: &[u8], right: &[u8]) -> bool {
let max_len = left.len().max(right.len());
let mut difference = left.len() ^ right.len();
for index in 0..max_len {
let left_byte = left.get(index).copied().unwrap_or_default();
let right_byte = right.get(index).copied().unwrap_or_default();
difference |= usize::from(left_byte ^ right_byte);
}
difference == 0
}
struct VsockListener {
fd: OwnedFd,
}
impl VsockListener {
fn bind(port: u32) -> io::Result<Self> {
let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|_| {
io::Error::new(io::ErrorKind::InvalidInput, "AF_VSOCK exceeds sa_family_t")
})?;
let address_length =
libc::socklen_t::try_from(size_of::<libc::sockaddr_vm>()).map_err(|_| {
io::Error::new(io::ErrorKind::InvalidInput, "sockaddr_vm exceeds socklen_t")
})?;
let raw_fd =
unsafe { libc::socket(libc::AF_VSOCK, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) };
if raw_fd < 0 {
return Err(io::Error::last_os_error());
}
let fd = unsafe { OwnedFd::from_raw_fd(raw_fd) };
let address = libc::sockaddr_vm {
svm_family: family,
svm_reserved1: 0,
svm_port: port,
svm_cid: libc::VMADDR_CID_ANY,
svm_zero: [0; 4],
};
let result = unsafe {
libc::bind(
fd.as_raw_fd(),
(&raw const address).cast::<libc::sockaddr>(),
address_length,
)
};
if result < 0 {
return Err(io::Error::last_os_error());
}
if unsafe { libc::listen(fd.as_raw_fd(), 16) } < 0 {
return Err(io::Error::last_os_error());
}
Ok(Self { fd })
}
fn accept(&self) -> io::Result<VsockStream> {
let raw_fd = unsafe {
libc::accept4(
self.fd.as_raw_fd(),
std::ptr::null_mut(),
std::ptr::null_mut(),
libc::SOCK_CLOEXEC,
)
};
if raw_fd < 0 {
Err(io::Error::last_os_error())
} else {
Ok(VsockStream {
file: unsafe { File::from_raw_fd(raw_fd) },
})
}
}
}
struct VsockStream {
file: File,
}
impl VsockStream {
fn set_timeout(&self, timeout: Duration) -> io::Result<()> {
let option_length =
libc::socklen_t::try_from(size_of::<libc::timeval>()).map_err(|_| {
io::Error::new(io::ErrorKind::InvalidInput, "timeval exceeds socklen_t")
})?;
let timeout = libc::timeval {
tv_sec: timeout.as_secs().try_into().unwrap_or(libc::time_t::MAX),
tv_usec: timeout.subsec_micros().into(),
};
for option in [libc::SO_RCVTIMEO, libc::SO_SNDTIMEO] {
let result = unsafe {
libc::setsockopt(
self.file.as_raw_fd(),
libc::SOL_SOCKET,
option,
(&raw const timeout).cast(),
option_length,
)
};
if result < 0 {
return Err(io::Error::last_os_error());
}
}
Ok(())
}
}
impl Read for VsockStream {
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
self.file.read(buffer)
}
}
impl Write for VsockStream {
fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {
self.file.write(buffer)
}
fn flush(&mut self) -> io::Result<()> {
self.file.flush()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn guest_config_debug_redacts_token() {
let config = GuestConfig {
boundary_id: "sandbox-1".to_string(),
bootstrap_token: "never-log-this-never-log-this".to_string(),
control_port: 5500,
agent_uid: DEFAULT_AGENT_UID,
agent_gid: DEFAULT_AGENT_GID,
};
let debug = format!("{config:?}");
assert!(debug.contains("<redacted>"));
assert!(!debug.contains("never-log-this"));
}
#[test]
fn constant_time_comparison_checks_length_and_content() {
assert!(constant_time_eq(b"same", b"same"));
assert!(!constant_time_eq(b"same", b"different"));
assert!(!constant_time_eq(b"same", b"sam"));
}
}
@@ -0,0 +1,17 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Experimental Firecracker Isolation Backend.
//!
//! The logical supervisor and RFC 0012 lifecycle live on the host. A private
//! guest mode of the same driver binary only transports lifecycle operations
//! and invokes the existing `openshell-supervisor-process` implementation.
pub mod backend;
pub mod guest;
mod protocol;
pub mod runtime;
pub use backend::{BACKEND_NAME, FirecrackerHostBackend, FirecrackerTopology};
pub use guest::{DEFAULT_GUEST_CONFIG_PATH, GuestConfig, run_guest};
pub use runtime::{FirecrackerLaunchConfig, FirecrackerVm};
@@ -0,0 +1,242 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use std::path::{Path, PathBuf};
use std::sync::Arc;
use clap::{Args, Parser, Subcommand};
use openshell_core::policy::{
FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy,
};
use openshell_driver_firecracker::{
BACKEND_NAME, DEFAULT_GUEST_CONFIG_PATH, FirecrackerHostBackend, FirecrackerLaunchConfig,
FirecrackerTopology, FirecrackerVm, run_guest,
};
use openshell_isolation::AgentSpec;
use openshell_isolation::contract::{
BackendRegistry, BoundaryExitStatus, INTERFACE_VERSION, SandboxContext, TopologyDescriptor,
};
#[derive(Debug, Parser)]
#[command(about = "Experimental OpenShell Firecracker isolation driver")]
struct Cli {
#[command(subcommand)]
command: Option<Command>,
}
#[derive(Debug, Subcommand)]
enum Command {
/// Boot and wait for one Firecracker VM.
Launch(LaunchArgs),
/// Drive the RFC 0012 lifecycle from the host.
Supervise(SuperviseArgs),
/// Run the private process-supervisor leaf transport inside the guest.
Guest(GuestArgs),
}
#[derive(Debug, Args)]
struct LaunchArgs {
#[arg(
long,
env = "OPENSHELL_FIRECRACKER_BINARY",
default_value = "firecracker"
)]
firecracker_binary: PathBuf,
#[arg(long)]
kernel_image: PathBuf,
#[arg(long)]
root_disk: PathBuf,
#[arg(long)]
run_dir: PathBuf,
#[arg(long)]
console_output: PathBuf,
#[arg(
long,
default_value = "/opt/openshell/bin/openshell-driver-firecracker"
)]
guest_init: String,
#[arg(long, default_value_t = 2)]
vcpus: u8,
#[arg(long, default_value_t = 512)]
mem_mib: u32,
#[arg(long)]
vsock_cid: u32,
}
#[derive(Debug, Args)]
struct SuperviseArgs {
#[arg(long)]
boundary_id: String,
#[arg(long)]
vsock_uds_path: PathBuf,
#[arg(long, default_value_t = 5500)]
vsock_port: u32,
#[arg(long)]
bootstrap_token_file: PathBuf,
#[arg(long, default_value = "/sandbox")]
workdir: String,
#[arg(long, default_value_t = 300)]
timeout_seconds: u64,
#[arg(required = true, trailing_var_arg = true)]
command: Vec<String>,
}
#[derive(Debug, Args)]
struct GuestArgs {
#[arg(long, default_value = DEFAULT_GUEST_CONFIG_PATH)]
config: PathBuf,
}
fn main() {
if let Err(error) = run() {
eprintln!("Firecracker driver failed: {error}");
std::process::exit(1);
}
}
fn run() -> Result<(), String> {
let cli = Cli::parse();
let command = match cli.command {
Some(command) => command,
None if std::process::id() == 1 => Command::Guest(GuestArgs {
config: PathBuf::from(DEFAULT_GUEST_CONFIG_PATH),
}),
None => return Err("a subcommand is required outside a Firecracker guest".to_string()),
};
match command {
Command::Guest(args) => run_guest(&args.config),
Command::Launch(args) => launch(args),
Command::Supervise(args) => {
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|error| format!("create host runtime: {error}"))?;
runtime.block_on(supervise(args))
}
}
}
fn launch(args: LaunchArgs) -> Result<(), String> {
let config = FirecrackerLaunchConfig {
firecracker_binary: resolve_executable(&args.firecracker_binary)?,
kernel_image: args.kernel_image,
root_disk: args.root_disk,
run_dir: args.run_dir,
console_output: args.console_output,
guest_init: args.guest_init,
vcpus: args.vcpus,
mem_mib: args.mem_mib,
vsock_cid: args.vsock_cid,
};
let mut vm = FirecrackerVm::launch(&config)?;
let status = vm.wait()?;
if status.success() {
Ok(())
} else {
Err(format!("Firecracker exited with status {status}"))
}
}
async fn supervise(args: SuperviseArgs) -> Result<(), String> {
let bootstrap_token = std::fs::read_to_string(&args.bootstrap_token_file)
.map_err(|error| {
format!(
"read bootstrap token file {}: {error}",
args.bootstrap_token_file.display()
)
})?
.trim()
.to_string();
let topology = FirecrackerTopology {
boundary_id: args.boundary_id.clone(),
vsock_uds_path: args.vsock_uds_path,
control_port: args.vsock_port,
bootstrap_token,
};
let descriptor = TopologyDescriptor {
version: INTERFACE_VERSION,
backend_name: BACKEND_NAME.to_string(),
payload: topology.encode().map_err(|error| error.to_string())?,
};
let mut registry = BackendRegistry::new();
registry
.register(Arc::new(FirecrackerHostBackend))
.map_err(|error| error.to_string())?;
let (backend, verified) = registry
.resolve(descriptor, BACKEND_NAME)
.map_err(|error| error.to_string())?;
let (program, command_args) = args
.command
.split_first()
.ok_or_else(|| "agent command must not be empty".to_string())?;
let sandbox = SandboxContext {
sandbox_id: args.boundary_id,
policy: restrictive_host_policy(),
agent: AgentSpec {
program: program.clone(),
args: command_args.to_vec(),
workdir: Some(args.workdir),
timeout_secs: args.timeout_seconds,
interactive: false,
},
};
let bound = backend
.attach(verified, sandbox)
.await
.map_err(|error| error.to_string())?;
let ready = bound.confirm().await.map_err(|error| error.to_string())?;
let running = ready
.start_agent()
.await
.map_err(|error| error.to_string())?;
let process = running.agent();
let wait = process.wait();
tokio::pin!(wait);
let status = tokio::select! {
result = &mut wait => result.map_err(|error| error.to_string())?,
signal = tokio::signal::ctrl_c() => {
signal.map_err(|error| format!("wait for interrupt: {error}"))?;
process.terminate().await.map_err(|error| error.to_string())?;
process.wait().await.map_err(|error| error.to_string())?
}
};
println!("agent exited: {status:?}");
match status {
BoundaryExitStatus::Exited(0) => Ok(()),
_ => Err(format!("agent exited unsuccessfully: {status:?}")),
}
}
fn restrictive_host_policy() -> SandboxPolicy {
SandboxPolicy {
version: 1,
filesystem: FilesystemPolicy {
read_only: ["/usr", "/lib", "/proc", "/dev/urandom", "/etc", "/var/log"]
.into_iter()
.map(PathBuf::from)
.collect(),
read_write: ["/sandbox", "/tmp", "/dev/null"]
.into_iter()
.map(PathBuf::from)
.collect(),
include_workdir: true,
},
network: NetworkPolicy::default(),
landlock: LandlockPolicy::default(),
process: ProcessPolicy::default(),
}
}
fn resolve_executable(path: &Path) -> Result<PathBuf, String> {
if path.components().count() > 1 || path.is_absolute() {
return path
.is_file()
.then(|| path.to_path_buf())
.ok_or_else(|| format!("executable not found: {}", path.display()));
}
let search_path = std::env::var_os("PATH").unwrap_or_default();
std::env::split_paths(&search_path)
.map(|directory| directory.join(path))
.find(|candidate| candidate.is_file())
.ok_or_else(|| format!("executable not found on PATH: {}", path.display()))
}
@@ -0,0 +1,361 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Driver-private, length-delimited JSON protocol carried over virtio-vsock.
use std::fmt;
use std::io::{self, Read, Write};
use openshell_core::policy::{
FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkMode, NetworkPolicy,
ProcessPolicy, ProxyPolicy, SandboxPolicy,
};
use openshell_isolation::AgentSpec;
use openshell_isolation::contract::{BoundaryExitStatus, BoundarySignal};
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024;
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RequestEnvelope {
pub request_id: u64,
pub boundary_id: String,
pub bootstrap_token: String,
pub request: Request,
}
impl fmt::Debug for RequestEnvelope {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
formatter
.debug_struct("RequestEnvelope")
.field("request_id", &self.request_id)
.field("boundary_id", &self.boundary_id)
.field("bootstrap_token", &"<redacted>")
.field("request", &self.request)
.finish()
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "operation", rename_all = "snake_case")]
pub enum Request {
Attach,
Confirm,
StartAgent {
sandbox_id: String,
spec: AgentSpecWire,
policy: Box<SandboxPolicyWire>,
},
Wait {
process_id: String,
},
Signal {
process_id: String,
signal: SignalWire,
},
Terminate {
process_id: String,
},
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct ResponseEnvelope {
pub request_id: u64,
pub response: Response,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "result", rename_all = "snake_case")]
pub enum Response {
Attached,
Confirmed,
Started { process_id: String },
Exited { status: ExitStatusWire },
Signaled,
Terminated,
Error { kind: String, message: String },
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct AgentSpecWire {
pub program: String,
pub args: Vec<String>,
pub workdir: Option<String>,
pub timeout_secs: u64,
pub interactive: bool,
}
impl From<AgentSpec> for AgentSpecWire {
fn from(spec: AgentSpec) -> Self {
Self {
program: spec.program,
args: spec.args,
workdir: spec.workdir,
timeout_secs: spec.timeout_secs,
interactive: spec.interactive,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct SandboxPolicyWire {
pub version: u32,
pub read_only: Vec<std::path::PathBuf>,
pub read_write: Vec<std::path::PathBuf>,
pub include_workdir: bool,
pub network: NetworkModeWire,
pub proxy_addr: Option<std::net::SocketAddr>,
pub landlock: LandlockCompatibilityWire,
pub run_as_user: Option<String>,
pub run_as_group: Option<String>,
}
impl From<SandboxPolicy> for SandboxPolicyWire {
fn from(policy: SandboxPolicy) -> Self {
Self {
version: policy.version,
read_only: policy.filesystem.read_only,
read_write: policy.filesystem.read_write,
include_workdir: policy.filesystem.include_workdir,
network: NetworkModeWire::from(policy.network.mode),
proxy_addr: policy.network.proxy.and_then(|proxy| proxy.http_addr),
landlock: LandlockCompatibilityWire::from(policy.landlock.compatibility),
run_as_user: policy.process.run_as_user,
run_as_group: policy.process.run_as_group,
}
}
}
impl From<SandboxPolicyWire> for SandboxPolicy {
fn from(policy: SandboxPolicyWire) -> Self {
let proxy = matches!(policy.network, NetworkModeWire::Proxy).then_some(ProxyPolicy {
http_addr: policy.proxy_addr,
});
Self {
version: policy.version,
filesystem: FilesystemPolicy {
read_only: policy.read_only,
read_write: policy.read_write,
include_workdir: policy.include_workdir,
},
network: NetworkPolicy {
mode: policy.network.into(),
proxy,
},
landlock: LandlockPolicy {
compatibility: policy.landlock.into(),
},
process: ProcessPolicy {
run_as_user: policy.run_as_user,
run_as_group: policy.run_as_group,
},
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum NetworkModeWire {
Block,
Proxy,
Allow,
}
impl From<NetworkMode> for NetworkModeWire {
fn from(mode: NetworkMode) -> Self {
match mode {
NetworkMode::Block => Self::Block,
NetworkMode::Proxy => Self::Proxy,
NetworkMode::Allow => Self::Allow,
}
}
}
impl From<NetworkModeWire> for NetworkMode {
fn from(mode: NetworkModeWire) -> Self {
match mode {
NetworkModeWire::Block => Self::Block,
NetworkModeWire::Proxy => Self::Proxy,
NetworkModeWire::Allow => Self::Allow,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum LandlockCompatibilityWire {
BestEffort,
HardRequirement,
}
impl From<LandlockCompatibility> for LandlockCompatibilityWire {
fn from(compatibility: LandlockCompatibility) -> Self {
match compatibility {
LandlockCompatibility::BestEffort => Self::BestEffort,
LandlockCompatibility::HardRequirement => Self::HardRequirement,
}
}
}
impl From<LandlockCompatibilityWire> for LandlockCompatibility {
fn from(compatibility: LandlockCompatibilityWire) -> Self {
match compatibility {
LandlockCompatibilityWire::BestEffort => Self::BestEffort,
LandlockCompatibilityWire::HardRequirement => Self::HardRequirement,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum SignalWire {
Term,
Kill,
Int,
Hup,
}
impl From<BoundarySignal> for SignalWire {
fn from(signal: BoundarySignal) -> Self {
match signal {
BoundarySignal::Term => Self::Term,
BoundarySignal::Kill => Self::Kill,
BoundarySignal::Int => Self::Int,
BoundarySignal::Hup => Self::Hup,
}
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(tag = "kind", content = "value", rename_all = "snake_case")]
pub enum ExitStatusWire {
Exited(i32),
Signaled(i32),
}
impl From<ExitStatusWire> for BoundaryExitStatus {
fn from(status: ExitStatusWire) -> Self {
match status {
ExitStatusWire::Exited(code) => Self::Exited(code),
ExitStatusWire::Signaled(signal) => Self::Signaled(signal),
}
}
}
pub fn encode_frame<T: Serialize>(message: &T) -> Result<Vec<u8>, FrameError> {
let payload = serde_json::to_vec(message).map_err(FrameError::Serialize)?;
if payload.len() > MAX_CONTROL_FRAME_BYTES {
return Err(FrameError::TooLarge(payload.len()));
}
let length = u32::try_from(payload.len()).map_err(|_| FrameError::TooLarge(payload.len()))?;
let mut frame = Vec::with_capacity(4 + payload.len());
frame.extend_from_slice(&length.to_be_bytes());
frame.extend_from_slice(&payload);
Ok(frame)
}
pub fn decode_frame<T: DeserializeOwned>(frame: &[u8]) -> Result<T, FrameError> {
let header: [u8; 4] = frame
.get(..4)
.ok_or(FrameError::Truncated)?
.try_into()
.map_err(|_| FrameError::Truncated)?;
let declared = u32::from_be_bytes(header) as usize;
if declared > MAX_CONTROL_FRAME_BYTES {
return Err(FrameError::TooLarge(declared));
}
let payload = frame.get(4..).ok_or(FrameError::Truncated)?;
if payload.len() != declared {
return Err(FrameError::LengthMismatch {
declared,
actual: payload.len(),
});
}
serde_json::from_slice(payload).map_err(FrameError::Deserialize)
}
pub fn read_frame<T: DeserializeOwned>(reader: &mut impl Read) -> Result<T, FrameError> {
let mut header = [0_u8; 4];
reader.read_exact(&mut header)?;
let declared = u32::from_be_bytes(header) as usize;
if declared > MAX_CONTROL_FRAME_BYTES {
return Err(FrameError::TooLarge(declared));
}
let mut frame = Vec::with_capacity(4 + declared);
frame.extend_from_slice(&header);
frame.resize(4 + declared, 0);
reader.read_exact(&mut frame[4..])?;
decode_frame(&frame)
}
pub fn write_frame<T: Serialize>(writer: &mut impl Write, message: &T) -> Result<(), FrameError> {
let frame = encode_frame(message)?;
writer.write_all(&frame)?;
writer.flush()?;
Ok(())
}
#[derive(Debug, thiserror::Error)]
pub enum FrameError {
#[error("control frame is truncated")]
Truncated,
#[error("control frame is too large: {0} bytes")]
TooLarge(usize),
#[error("control frame declared {declared} bytes but contained {actual}")]
LengthMismatch { declared: usize, actual: usize },
#[error("serialize control frame: {0}")]
Serialize(serde_json::Error),
#[error("deserialize control frame: {0}")]
Deserialize(serde_json::Error),
#[error("read or write control frame: {0}")]
Io(#[from] io::Error),
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn request_round_trips_and_redacts_token() {
let request = RequestEnvelope {
request_id: 7,
boundary_id: "sandbox-1".to_string(),
bootstrap_token: "never-log-this".to_string(),
request: Request::StartAgent {
sandbox_id: "sandbox-1".to_string(),
spec: AgentSpecWire {
program: "/bin/true".to_string(),
args: Vec::new(),
workdir: Some("/sandbox".to_string()),
timeout_secs: 5,
interactive: false,
},
policy: Box::new(SandboxPolicyWire::from(SandboxPolicy {
version: 1,
filesystem: FilesystemPolicy::default(),
network: NetworkPolicy::default(),
landlock: LandlockPolicy::default(),
process: ProcessPolicy::default(),
})),
},
};
let frame = encode_frame(&request).expect("encode request");
let decoded: RequestEnvelope = decode_frame(&frame).expect("decode request");
assert_eq!(decoded, request);
let debug = format!("{request:?}");
assert!(debug.contains("<redacted>"));
assert!(!debug.contains("never-log-this"));
}
#[test]
fn rejects_declared_oversize() {
let oversized = u32::try_from(MAX_CONTROL_FRAME_BYTES + 1).expect("test size fits u32");
let mut frame = Vec::from(oversized.to_be_bytes());
frame.extend_from_slice(b"{}");
assert!(matches!(
decode_frame::<RequestEnvelope>(&frame),
Err(FrameError::TooLarge(_))
));
}
}
@@ -0,0 +1,401 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Small standalone Firecracker launcher owned by this driver crate.
#![allow(unsafe_code)]
use std::fs::{File, OpenOptions};
use std::io::{self, Read, Write};
use std::os::unix::net::UnixStream;
use std::os::unix::process::CommandExt as _;
use std::path::{Path, PathBuf};
use std::process::{Child, Command, ExitStatus, Stdio};
use std::time::{Duration, Instant};
use nix::sys::signal::Signal;
use serde_json::{Value, json};
const API_START_TIMEOUT: Duration = Duration::from_secs(5);
const API_IO_TIMEOUT: Duration = Duration::from_secs(5);
const MAX_API_RESPONSE_BYTES: usize = 1024 * 1024;
#[derive(Debug, Clone)]
pub struct FirecrackerLaunchConfig {
pub firecracker_binary: PathBuf,
pub kernel_image: PathBuf,
pub root_disk: PathBuf,
pub run_dir: PathBuf,
pub console_output: PathBuf,
pub guest_init: String,
pub vcpus: u8,
pub mem_mib: u32,
pub vsock_cid: u32,
}
/// A configured Firecracker child. Dropping it terminates the VMM.
pub struct FirecrackerVm {
child: Child,
api_socket: PathBuf,
vsock_socket: PathBuf,
}
impl FirecrackerVm {
pub fn launch(config: &FirecrackerLaunchConfig) -> Result<Self, String> {
validate_config(config)?;
check_kvm_access()?;
std::fs::create_dir_all(&config.run_dir).map_err(|error| {
format!(
"create Firecracker run dir {}: {error}",
config.run_dir.display()
)
})?;
let api_socket = config.run_dir.join("firecracker-api.sock");
let vsock_socket = config.run_dir.join("firecracker-vsock.sock");
remove_stale_socket(&api_socket)?;
remove_stale_socket(&vsock_socket)?;
let console = File::create(&config.console_output).map_err(|error| {
format!(
"create Firecracker console log {}: {error}",
config.console_output.display()
)
})?;
let stderr = console
.try_clone()
.map_err(|error| format!("clone Firecracker console log: {error}"))?;
let mut command = Command::new(&config.firecracker_binary);
command
.arg("--api-sock")
.arg(&api_socket)
.stdin(Stdio::null())
.stdout(Stdio::from(console))
.stderr(Stdio::from(stderr));
unsafe {
command.pre_exec(|| {
nix::sys::prctl::set_pdeathsig(Signal::SIGKILL)
.map_err(|error| io::Error::other(format!("pdeathsig: {error}")))
});
}
let mut child = command
.spawn()
.map_err(|error| format!("start Firecracker: {error}"))?;
if let Err(error) = configure(&mut child, &api_socket, &vsock_socket, config) {
terminate_child(&mut child);
return Err(error);
}
Ok(Self {
child,
api_socket,
vsock_socket,
})
}
pub fn vsock_uds_path(&self) -> &Path {
&self.vsock_socket
}
pub fn wait(&mut self) -> Result<ExitStatus, String> {
self.child
.wait()
.map_err(|error| format!("wait for Firecracker: {error}"))
}
pub fn terminate(&mut self) -> Result<(), String> {
terminate_child(&mut self.child);
Ok(())
}
}
impl Drop for FirecrackerVm {
fn drop(&mut self) {
if self.child.try_wait().ok().flatten().is_none() {
terminate_child(&mut self.child);
}
let _ = std::fs::remove_file(&self.api_socket);
let _ = std::fs::remove_file(&self.vsock_socket);
}
}
fn validate_config(config: &FirecrackerLaunchConfig) -> Result<(), String> {
for (label, path) in [
("Firecracker binary", &config.firecracker_binary),
("kernel image", &config.kernel_image),
("root disk", &config.root_disk),
] {
if !path.is_file() {
return Err(format!("{label} not found: {}", path.display()));
}
}
if config.vcpus == 0 {
return Err("Firecracker vCPU count must be nonzero".to_string());
}
if config.mem_mib < 128 {
return Err("Firecracker memory must be at least 128 MiB".to_string());
}
if config.vsock_cid < 3 {
return Err("Firecracker guest CID must be at least 3".to_string());
}
if !config.guest_init.starts_with('/') {
return Err("Firecracker guest init path must be absolute".to_string());
}
Ok(())
}
fn check_kvm_access() -> Result<(), String> {
OpenOptions::new()
.read(true)
.write(true)
.open("/dev/kvm")
.map(|_| ())
.map_err(|error| {
format!(
"open /dev/kvm read/write: {error}; start a login session whose supplementary groups include kvm"
)
})
}
fn configure(
child: &mut Child,
api_socket: &Path,
vsock_socket: &Path,
config: &FirecrackerLaunchConfig,
) -> Result<(), String> {
wait_for_api_socket(child, api_socket)?;
for request in configuration_requests(config, vsock_socket) {
put_json(api_socket, request.path, &request.body)?;
}
Ok(())
}
struct ApiRequest {
path: &'static str,
body: Value,
}
fn configuration_requests(
config: &FirecrackerLaunchConfig,
vsock_socket: &Path,
) -> Vec<ApiRequest> {
vec![
ApiRequest {
path: "/machine-config",
body: json!({
"vcpu_count": config.vcpus,
"mem_size_mib": config.mem_mib,
"smt": false
}),
},
ApiRequest {
path: "/boot-source",
body: json!({
"kernel_image_path": config.kernel_image,
"boot_args": kernel_command_line(&config.guest_init)
}),
},
ApiRequest {
path: "/drives/rootfs",
body: json!({
"drive_id": "rootfs",
"path_on_host": config.root_disk,
"is_root_device": true,
"is_read_only": false
}),
},
ApiRequest {
path: "/vsock",
body: json!({
"guest_cid": config.vsock_cid,
"uds_path": vsock_socket
}),
},
ApiRequest {
path: "/actions",
body: json!({ "action_type": "InstanceStart" }),
},
]
}
fn kernel_command_line(init: &str) -> String {
format!("console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw init={init}")
}
fn wait_for_api_socket(child: &mut Child, socket: &Path) -> Result<(), String> {
let deadline = Instant::now() + API_START_TIMEOUT;
loop {
if UnixStream::connect(socket).is_ok() {
return Ok(());
}
if let Some(status) = child
.try_wait()
.map_err(|error| format!("check Firecracker process: {error}"))?
{
return Err(format!(
"Firecracker exited before its API socket was ready: {status}"
));
}
if Instant::now() >= deadline {
return Err(format!(
"timed out waiting for Firecracker API socket {}",
socket.display()
));
}
std::thread::sleep(Duration::from_millis(20));
}
}
fn put_json(socket: &Path, path: &str, body: &Value) -> Result<(), String> {
let body = serde_json::to_vec(body).map_err(|error| format!("encode {path}: {error}"))?;
let mut stream = UnixStream::connect(socket)
.map_err(|error| format!("connect to Firecracker API {}: {error}", socket.display()))?;
stream
.set_read_timeout(Some(API_IO_TIMEOUT))
.map_err(|error| format!("set API read timeout: {error}"))?;
stream
.set_write_timeout(Some(API_IO_TIMEOUT))
.map_err(|error| format!("set API write timeout: {error}"))?;
write!(
stream,
"PUT {path} HTTP/1.1\r\nHost: localhost\r\nAccept: application/json\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
body.len()
)
.and_then(|()| stream.write_all(&body))
.map_err(|error| format!("write Firecracker API request {path}: {error}"))?;
let response = read_http_response(&mut stream, path)?;
check_http_status(path, &response)
}
fn read_http_response(stream: &mut UnixStream, path: &str) -> Result<Vec<u8>, String> {
let mut response = Vec::with_capacity(1024);
let mut buffer = [0_u8; 4096];
loop {
let count = stream
.read(&mut buffer)
.map_err(|error| format!("read Firecracker API response {path}: {error}"))?;
if count == 0 {
return if response.is_empty() {
Err(format!("empty Firecracker API response for {path}"))
} else {
Ok(response)
};
}
response.extend_from_slice(&buffer[..count]);
if response.len() > MAX_API_RESPONSE_BYTES {
return Err(format!(
"Firecracker API response for {path} exceeds {MAX_API_RESPONSE_BYTES} bytes"
));
}
let Some(header_end) = response.windows(4).position(|window| window == b"\r\n\r\n") else {
continue;
};
let body_start = header_end + 4;
let headers = String::from_utf8_lossy(&response[..header_end]);
let content_length = headers.lines().find_map(|line| {
let (name, value) = line.split_once(':')?;
name.eq_ignore_ascii_case("content-length")
.then(|| value.trim().parse::<usize>().ok())
.flatten()
});
match content_length {
Some(length) if response.len() >= body_start + length => {
response.truncate(body_start + length);
return Ok(response);
}
Some(_) => {}
None => return Ok(response),
}
}
}
fn check_http_status(path: &str, response: &[u8]) -> Result<(), String> {
let text = String::from_utf8_lossy(response);
let status_line = text
.lines()
.next()
.ok_or_else(|| format!("empty Firecracker API response for {path}"))?;
let status = status_line
.split_whitespace()
.nth(1)
.and_then(|value| value.parse::<u16>().ok())
.ok_or_else(|| format!("invalid Firecracker API response for {path}: {status_line}"))?;
if (200..300).contains(&status) {
Ok(())
} else {
let body = text.split_once("\r\n\r\n").map_or("", |(_, body)| body);
Err(format!(
"Firecracker API request {path} failed with status {status}: {}",
body.trim()
))
}
}
fn remove_stale_socket(path: &Path) -> Result<(), String> {
match std::fs::remove_file(path) {
Ok(()) => Ok(()),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(format!("remove stale socket {}: {error}", path.display())),
}
}
fn terminate_child(child: &mut Child) {
let _ = child.kill();
let _ = child.wait();
}
#[cfg(test)]
mod tests {
use super::*;
fn config() -> FirecrackerLaunchConfig {
FirecrackerLaunchConfig {
firecracker_binary: PathBuf::from("/firecracker"),
kernel_image: PathBuf::from("/vmlinux"),
root_disk: PathBuf::from("/root.ext4"),
run_dir: PathBuf::from("/run/firecracker"),
console_output: PathBuf::from("/run/firecracker/console.log"),
guest_init: "/opt/openshell/bin/openshell-driver-firecracker".to_string(),
vcpus: 2,
mem_mib: 512,
vsock_cid: 4,
}
}
#[test]
fn configures_no_network_device() {
let config = config();
let requests = configuration_requests(&config, Path::new("/tmp/vsock.sock"));
let paths = requests
.iter()
.map(|request| request.path)
.collect::<Vec<_>>();
assert_eq!(
paths,
[
"/machine-config",
"/boot-source",
"/drives/rootfs",
"/vsock",
"/actions"
]
);
assert!(
!requests
.iter()
.any(|request| request.path == "/network-interfaces")
);
assert_eq!(requests[2].body["is_read_only"], false);
}
#[test]
fn boot_source_uses_driver_as_init() {
let command_line = kernel_command_line("/opt/openshell/bin/openshell-driver-firecracker");
assert!(command_line.contains("root=/dev/vda rw"));
assert!(command_line.contains("init=/opt/openshell/bin/openshell-driver-firecracker"));
}
#[test]
fn accepts_only_successful_api_statuses() {
assert!(check_http_status("/actions", b"HTTP/1.1 204 No Content\r\n\r\n").is_ok());
assert!(check_http_status("/actions", b"HTTP/1.1 400 Bad\r\n\r\nnope").is_err());
}
}
+23
View File
@@ -0,0 +1,23 @@
# Firecracker smoke test
This runner boots a real Firecracker VM, drives the RFC 0012 lifecycle from the
host, and verifies that the guest process leaf runs the admitted command through
the existing OpenShell process supervisor.
It does not use `sudo`, a TAP interface, a guest NIC, nftables, or
`CAP_NET_ADMIN`. The current login session must have read/write access to
`/dev/kvm`.
Default fixtures live in `/tmp/openshell-firecracker-e2e-fixtures`. Override
them with:
```shell
OPENSHELL_FIRECRACKER_BINARY=/path/to/firecracker \
OPENSHELL_FIRECRACKER_KERNEL_IMAGE=/path/to/vmlinux \
OPENSHELL_FIRECRACKER_ROOT_DISK=/path/to/root.ext4 \
mise run e2e:firecracker
```
The runner clones the root disk into a temporary directory and injects the
current `openshell-driver-firecracker` binary plus a one-time authenticated
guest configuration. It never modifies the source fixture.
+174
View File
@@ -0,0 +1,174 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Boot the host-supervised Firecracker backend and exercise its RFC 0012
# lifecycle against the current process supervisor implementation in the guest.
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
FIXTURE_DIR="${OPENSHELL_FIRECRACKER_FIXTURE_DIR:-/tmp/openshell-firecracker-e2e-fixtures}"
DRIVER_BIN="${OPENSHELL_FIRECRACKER_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-firecracker}"
FIRECRACKER_BIN="${OPENSHELL_FIRECRACKER_BINARY:-${FIXTURE_DIR}/release/release-v1.16.1-aarch64/firecracker-v1.16.1-aarch64}"
KERNEL_IMAGE="${OPENSHELL_FIRECRACKER_KERNEL_IMAGE:-${FIXTURE_DIR}/vmlinux-6.1.155}"
ROOT_DISK_FIXTURE="${OPENSHELL_FIRECRACKER_ROOT_DISK:-${FIXTURE_DIR}/openshell-ubuntu-24.04.ext4}"
BOOT_TIMEOUT_SECONDS="${OPENSHELL_FIRECRACKER_BOOT_TIMEOUT_SECONDS:-60}"
KEEP_STATE="${OPENSHELL_FIRECRACKER_KEEP_STATE:-0}"
CONTROL_PORT="${OPENSHELL_FIRECRACKER_CONTROL_PORT:-5500}"
RUN_DIR="$(mktemp -d /tmp/openshell-firecracker-e2e.XXXXXX)"
ROOT_DISK="${RUN_DIR}/root.ext4"
TOKEN_FILE="${RUN_DIR}/bootstrap.token"
GUEST_CONFIG="${RUN_DIR}/firecracker.json"
CONSOLE_LOG="${RUN_DIR}/console.log"
LAUNCHER_LOG="${RUN_DIR}/launcher.log"
SUPERVISOR_LOG="${RUN_DIR}/supervisor.log"
VSOCK_SOCKET="${RUN_DIR}/firecracker-vsock.sock"
LAUNCHER_PID=""
BOUNDARY_ID="firecracker-e2e-$$"
VSOCK_CID=$(( ($$ % 60000) + 1024 ))
SMOKE_MARKER="openshell-firecracker-e2e-ok-${BOUNDARY_ID}"
fail() {
echo "ERROR: $*" >&2
exit 1
}
launcher_is_running() {
[ -n "${LAUNCHER_PID}" ] && kill -0 "${LAUNCHER_PID}" 2>/dev/null
}
print_logs() {
local log
for log in "${LAUNCHER_LOG}" "${SUPERVISOR_LOG}" "${CONSOLE_LOG}"; do
if [ -s "${log}" ]; then
echo "=== ${log} ===" >&2
tail -n 200 "${log}" >&2 || true
echo "=== end ${log} ===" >&2
fi
done
}
cleanup() {
local exit_code=$?
set +e
if launcher_is_running; then
kill -TERM "${LAUNCHER_PID}" 2>/dev/null
for _ in 1 2 3 4 5 6 7 8 9 10; do
launcher_is_running || break
sleep 0.2
done
launcher_is_running && kill -KILL "${LAUNCHER_PID}" 2>/dev/null
fi
[ -n "${LAUNCHER_PID}" ] && wait "${LAUNCHER_PID}" 2>/dev/null
[ "${exit_code}" -eq 0 ] || print_logs
if [ "${KEEP_STATE}" = "1" ]; then
echo "Firecracker E2E state preserved at ${RUN_DIR}" >&2
else
case "${RUN_DIR}" in
/tmp/openshell-firecracker-e2e.*) rm -rf -- "${RUN_DIR}" ;;
*) echo "Refusing to remove unexpected run directory: ${RUN_DIR}" >&2 ;;
esac
fi
exit "${exit_code}"
}
trap cleanup EXIT
wait_for_console_marker() {
local marker="$1"
local label="$2"
local deadline=$(( SECONDS + BOOT_TIMEOUT_SECONDS ))
while [ "${SECONDS}" -lt "${deadline}" ]; do
grep -Fq -- "${marker}" "${CONSOLE_LOG}" 2>/dev/null && return 0
launcher_is_running || fail "Firecracker exited while waiting for ${label}"
sleep 0.2
done
fail "timed out after ${BOOT_TIMEOUT_SECONDS}s waiting for ${label}"
}
inject_file() {
local source="$1"
local destination="$2"
debugfs -w -R "rm ${destination}" "${ROOT_DISK}" >/dev/null 2>&1 || true
debugfs -w -R "write ${source} ${destination}" "${ROOT_DISK}" >/dev/null
}
[ "$(uname -s)" = "Linux" ] || fail "Firecracker E2E requires Linux"
[ -r /dev/kvm ] && [ -w /dev/kvm ] || \
fail "/dev/kvm is not readable and writable; start a login session with the kvm supplementary group"
[ -x "${FIRECRACKER_BIN}" ] || fail "Firecracker binary is not executable: ${FIRECRACKER_BIN}"
[ -f "${KERNEL_IMAGE}" ] || fail "kernel image not found: ${KERNEL_IMAGE}"
[ -f "${ROOT_DISK_FIXTURE}" ] || fail "root disk fixture not found: ${ROOT_DISK_FIXTURE}"
[[ "${BOOT_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]] || fail "boot timeout must be positive"
[[ "${CONTROL_PORT}" =~ ^[1-9][0-9]*$ ]] || fail "control port must be positive"
for tool in cargo cp debugfs grep jq openssl tail; do
command -v "${tool}" >/dev/null 2>&1 || fail "required host tool not found: ${tool}"
done
if [ -n "${RUSTC_WRAPPER:-}" ] && [ "${OPENSHELL_E2E_FIRECRACKER_ALLOW_RUSTC_WRAPPER:-0}" != "1" ]; then
unset RUSTC_WRAPPER
fi
if [ -z "${OPENSHELL_FIRECRACKER_DRIVER_BIN:-}" ]; then
echo "==> Building standalone Firecracker driver"
cargo build --package openshell-driver-firecracker
fi
[ -x "${DRIVER_BIN}" ] || fail "driver binary not found: ${DRIVER_BIN}"
echo "==> Preparing an isolated guest disk"
cp --reflink=auto "${ROOT_DISK_FIXTURE}" "${ROOT_DISK}"
umask 077
openssl rand -hex 32 >"${TOKEN_FILE}"
jq -n \
--arg boundary_id "${BOUNDARY_ID}" \
--arg bootstrap_token "$(tr -d '\n' <"${TOKEN_FILE}")" \
--argjson control_port "${CONTROL_PORT}" \
'{boundary_id: $boundary_id, bootstrap_token: $bootstrap_token, control_port: $control_port}' \
>"${GUEST_CONFIG}"
debugfs -w -R "mkdir /etc/openshell" "${ROOT_DISK}" >/dev/null 2>&1 || true
inject_file "${GUEST_CONFIG}" /etc/openshell/firecracker.json
inject_file "${DRIVER_BIN}" /opt/openshell/bin/openshell-driver-firecracker
debugfs -w -R \
"set_inode_field /opt/openshell/bin/openshell-driver-firecracker mode 0100755" \
"${ROOT_DISK}" >/dev/null
echo "==> Starting Firecracker guest ${BOUNDARY_ID} (vsock only, no NIC)"
"${DRIVER_BIN}" launch \
--firecracker-binary "${FIRECRACKER_BIN}" \
--kernel-image "${KERNEL_IMAGE}" \
--root-disk "${ROOT_DISK}" \
--run-dir "${RUN_DIR}" \
--console-output "${CONSOLE_LOG}" \
--vsock-cid "${VSOCK_CID}" \
>"${LAUNCHER_LOG}" 2>&1 &
LAUNCHER_PID=$!
wait_for_console_marker \
"Firecracker process supervisor leaf listening on vsock port ${CONTROL_PORT}" \
"guest process leaf"
[ -S "${VSOCK_SOCKET}" ] || fail "Firecracker vsock UDS was not created: ${VSOCK_SOCKET}"
echo "==> Driving attach, confirm, start_agent, and wait from the host"
"${DRIVER_BIN}" supervise \
--boundary-id "${BOUNDARY_ID}" \
--vsock-uds-path "${VSOCK_SOCKET}" \
--vsock-port "${CONTROL_PORT}" \
--bootstrap-token-file "${TOKEN_FILE}" \
--workdir /sandbox \
--timeout-seconds 30 \
-- /bin/sh -lc "printf '%s\\n' '${SMOKE_MARKER}'" \
>"${SUPERVISOR_LOG}" 2>&1
grep -Fq 'agent exited: Exited(0)' "${SUPERVISOR_LOG}" || \
fail "host supervisor did not observe a successful agent exit"
wait_for_console_marker "${SMOKE_MARKER}" "agent output"
echo "==> Firecracker host-supervisor E2E passed"
echo " boundary: ${BOUNDARY_ID}"
echo " lifecycle: attach -> confirm -> start_agent -> wait"
echo " process leaf: openshell-supervisor-process"
echo " transport: authenticated Firecracker virtio-vsock"
echo " network: no guest NIC"
+4
View File
@@ -155,6 +155,10 @@ run = "e2e/rust/e2e-kubernetes.sh"
description = "Start openshell-gateway with the VM compute driver and run VM e2e tests"
run = "e2e/rust/e2e-vm.sh"
["e2e:firecracker"]
description = "Boot Firecracker and run the host-supervised RFC 0012 lifecycle smoke test"
run = "e2e/firecracker/smoke.sh"
["e2e:docker"]
description = "Run smoke e2e against a standalone gateway with the Docker compute driver"
run = "e2e/rust/e2e-docker.sh"