mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-21 06:11:31 +08:00
feat(firecracker): prototype host-supervised isolation
Signed-off-by: Drew Newberry <385+drew@users.noreply.github.com>
This commit is contained in:
@@ -47,6 +47,7 @@ These pipelines connect skills into end-to-end workflows. Individual skill files
|
||||
| `crates/openshell-driver-docker/` | Docker compute driver | In-process `ComputeDriver` backend for local Docker sandbox containers |
|
||||
| `crates/openshell-driver-podman/` | Podman compute driver | In-process `ComputeDriver` backend for local Podman sandbox containers |
|
||||
| `crates/openshell-driver-vm/` | VM compute driver | Standalone libkrun-backed `ComputeDriver` subprocess (embeds its own rootfs + runtime) |
|
||||
| `crates/openshell-driver-firecracker/` | Firecracker isolation driver | Experimental host-side RFC 0012 backend with a private guest process-supervisor leaf mode |
|
||||
| `crates/openshell-prover/` | Policy prover | Policy verification and proof generation |
|
||||
| `crates/openshell-server-macros/` | Server macros | Compile-time helpers for gateway RPC authorization |
|
||||
| `crates/openshell-supervisor-middleware/` | Middleware runtime | Generic middleware registry, remote service integration, and chain execution |
|
||||
|
||||
Generated
+18
@@ -3920,6 +3920,24 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openshell-driver-firecracker"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"clap",
|
||||
"libc",
|
||||
"nix 0.29.0",
|
||||
"openshell-core",
|
||||
"openshell-isolation",
|
||||
"openshell-supervisor-process",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tempfile",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "openshell-driver-kubernetes"
|
||||
version = "0.0.0"
|
||||
|
||||
@@ -158,6 +158,7 @@ Runtime-specific implementation notes belong in the driver crate README:
|
||||
- `crates/openshell-driver-podman/README.md`
|
||||
- `crates/openshell-driver-kubernetes/README.md`
|
||||
- `crates/openshell-driver-vm/README.md`
|
||||
- `crates/openshell-driver-firecracker/README.md`
|
||||
|
||||
The combined VM topology runs `openshell-sandbox` as guest PID 1. libkrun
|
||||
executes the driver-owned guest bootstrap as PID 1, and the bootstrap preserves
|
||||
|
||||
@@ -70,6 +70,14 @@ by default. The co-located backend requires the
|
||||
supervisor to own the execution environment's PID namespace so boundary
|
||||
teardown can terminate every remaining workload process.
|
||||
|
||||
The experimental Firecracker backend exercises a delegated placement. The
|
||||
logical supervisor stays on the host and sends the admitted policy and workload
|
||||
spec over an authenticated, backend-private virtio-vsock channel only after the
|
||||
RFC lifecycle reaches `start_agent`. A private guest mode of the Firecracker
|
||||
driver invokes the existing process-supervisor implementation as the in-VM
|
||||
leaf. The prototype attaches no guest NIC, so network access is blocked by the
|
||||
VM topology while mediated egress remains unimplemented.
|
||||
|
||||
For proxy-mode boundaries, the co-located backend verifies its default-deny
|
||||
kernel egress ceiling before exposing any workload execution surface and then
|
||||
rechecks it every 250 milliseconds. Each check has a two-second deadline.
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
[package]
|
||||
name = "openshell-driver-firecracker"
|
||||
description = "Experimental Firecracker isolation driver for OpenShell"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lib]
|
||||
name = "openshell_driver_firecracker"
|
||||
path = "src/lib.rs"
|
||||
|
||||
[[bin]]
|
||||
name = "openshell-driver-firecracker"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
openshell-isolation = { path = "../openshell-isolation" }
|
||||
openshell-supervisor-process = { path = "../openshell-supervisor-process" }
|
||||
|
||||
async-trait = "0.1"
|
||||
clap = { workspace = true }
|
||||
libc = "0.2"
|
||||
nix = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
@@ -0,0 +1,37 @@
|
||||
# OpenShell Firecracker driver
|
||||
|
||||
`openshell-driver-firecracker` is an experimental host-side implementation of
|
||||
the RFC 0012 Isolation Backend contract. It deliberately lives outside the
|
||||
existing libkrun VM driver so the two runtimes can evolve independently.
|
||||
|
||||
The main OpenShell supervisor remains on the host. A private mode of the same
|
||||
driver binary listens on the guest's virtio-vsock device and invokes the current
|
||||
`openshell-supervisor-process` implementation only after the host advances the
|
||||
boundary through `attach -> confirm -> start_agent`. The admitted policy crosses
|
||||
that authenticated channel with the workload spec. The private protocol is not
|
||||
a second public supervisor or agent API.
|
||||
|
||||
The prototype has no virtual NIC. This makes the network ceiling structurally
|
||||
fail closed without TAP devices, nftables, `CAP_NET_ADMIN`, or `sudo`. Host
|
||||
requirements are a Linux Firecracker binary and read/write access to `/dev/kvm`.
|
||||
|
||||
Current scope:
|
||||
|
||||
- boots an existing ext4 guest image with Firecracker;
|
||||
- authenticates host-to-guest control over virtio-vsock;
|
||||
- implements the RFC lifecycle and agent wait/signal operations;
|
||||
- delegates guest process enforcement to the existing process supervisor leaf;
|
||||
- provides an unprivileged KVM end-to-end smoke runner.
|
||||
|
||||
Exec, PTY, port forwarding, mediated guest egress, and per-connection binary
|
||||
identity are intentionally deferred. The contract surfaces fail closed for
|
||||
those operations. Code that might later become a shared VM helper is duplicated
|
||||
here until a second consumer establishes a small, stable abstraction.
|
||||
|
||||
Run the smoke test with:
|
||||
|
||||
```shell
|
||||
mise run e2e:firecracker
|
||||
```
|
||||
|
||||
See `e2e/firecracker/README.md` for fixture overrides.
|
||||
@@ -0,0 +1,493 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Host-side RFC 0012 backend for an already-provisioned Firecracker VM.
|
||||
|
||||
use std::fmt;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use openshell_isolation::AgentSpec;
|
||||
use openshell_isolation::contract::{
|
||||
BackendError, BoundBoundary, BoundaryDuplexStream, BoundaryExec, BoundaryExitStatus,
|
||||
BoundaryPortForward, BoundaryProcess, BoundarySignal, ExecSession, ExecSpec, INTERFACE_VERSION,
|
||||
IsolationBackend, LoopbackTarget, MediatedConnection, NetworkMediationSource, ReadyBoundary,
|
||||
RunningBoundary, SandboxContext, VerifiedTopologyDescriptor,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
use crate::protocol::{
|
||||
AgentSpecWire, MAX_CONTROL_FRAME_BYTES, Request, RequestEnvelope, Response, ResponseEnvelope,
|
||||
SandboxPolicyWire, SignalWire, decode_frame, encode_frame,
|
||||
};
|
||||
|
||||
pub const BACKEND_NAME: &str = "firecracker";
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MIN_BOOTSTRAP_TOKEN_BYTES: usize = 32;
|
||||
const MAX_VSOCK_ACK_BYTES: usize = 64;
|
||||
|
||||
/// Backend-private provisioned topology payload.
|
||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct FirecrackerTopology {
|
||||
pub boundary_id: String,
|
||||
pub vsock_uds_path: PathBuf,
|
||||
pub control_port: u32,
|
||||
pub bootstrap_token: String,
|
||||
}
|
||||
|
||||
impl fmt::Debug for FirecrackerTopology {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("FirecrackerTopology")
|
||||
.field("boundary_id", &self.boundary_id)
|
||||
.field("vsock_uds_path", &self.vsock_uds_path)
|
||||
.field("control_port", &self.control_port)
|
||||
.field("bootstrap_token", &"<redacted>")
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl FirecrackerTopology {
|
||||
pub fn encode(&self) -> Result<Vec<u8>, BackendError> {
|
||||
serde_json::to_vec(self)
|
||||
.map_err(|error| BackendError::Descriptor(format!("encode topology: {error}")))
|
||||
}
|
||||
}
|
||||
|
||||
/// Host-side Firecracker implementation registered with the supervisor.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct FirecrackerHostBackend;
|
||||
|
||||
#[async_trait]
|
||||
impl IsolationBackend for FirecrackerHostBackend {
|
||||
fn backend_name(&self) -> &str {
|
||||
BACKEND_NAME
|
||||
}
|
||||
|
||||
fn version(&self) -> u32 {
|
||||
INTERFACE_VERSION
|
||||
}
|
||||
|
||||
async fn attach(
|
||||
&self,
|
||||
descriptor: VerifiedTopologyDescriptor,
|
||||
sandbox: SandboxContext,
|
||||
) -> Result<Box<dyn BoundBoundary>, BackendError> {
|
||||
let topology: FirecrackerTopology = serde_json::from_slice(descriptor.payload())
|
||||
.map_err(|error| BackendError::Descriptor(format!("decode topology: {error}")))?;
|
||||
validate_topology(&topology, &sandbox)?;
|
||||
let client = Arc::new(GuestClient::new(topology));
|
||||
expect_response(client.call(Request::Attach).await?, "attached")?;
|
||||
Ok(Box::new(FirecrackerBound {
|
||||
client,
|
||||
agent: sandbox.agent,
|
||||
policy: sandbox.policy,
|
||||
sandbox_id: sandbox.sandbox_id,
|
||||
mediation: Arc::new(NoGuestNetwork),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_topology(
|
||||
topology: &FirecrackerTopology,
|
||||
sandbox: &SandboxContext,
|
||||
) -> Result<(), BackendError> {
|
||||
if topology.boundary_id != sandbox.sandbox_id {
|
||||
return Err(BackendError::Descriptor(format!(
|
||||
"Firecracker boundary {:?} does not match sandbox {:?}",
|
||||
topology.boundary_id, sandbox.sandbox_id
|
||||
)));
|
||||
}
|
||||
if topology.bootstrap_token.len() < MIN_BOOTSTRAP_TOKEN_BYTES {
|
||||
return Err(BackendError::Descriptor(format!(
|
||||
"Firecracker bootstrap token must be at least {MIN_BOOTSTRAP_TOKEN_BYTES} bytes"
|
||||
)));
|
||||
}
|
||||
if !topology.vsock_uds_path.is_absolute() {
|
||||
return Err(BackendError::Descriptor(
|
||||
"Firecracker vsock UDS path must be absolute".to_string(),
|
||||
));
|
||||
}
|
||||
if topology.control_port == 0 {
|
||||
return Err(BackendError::Descriptor(
|
||||
"Firecracker guest control port must be nonzero".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct FirecrackerBound {
|
||||
client: Arc<GuestClient>,
|
||||
agent: AgentSpec,
|
||||
policy: openshell_core::policy::SandboxPolicy,
|
||||
sandbox_id: String,
|
||||
mediation: Arc<NoGuestNetwork>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BoundBoundary for FirecrackerBound {
|
||||
fn network_mediation_source(&self) -> Arc<dyn NetworkMediationSource> {
|
||||
self.mediation.clone()
|
||||
}
|
||||
|
||||
async fn confirm(self: Box<Self>) -> Result<Box<dyn ReadyBoundary>, BackendError> {
|
||||
expect_response(self.client.call(Request::Confirm).await?, "confirmed")?;
|
||||
Ok(Box::new(FirecrackerReady {
|
||||
client: self.client,
|
||||
agent: self.agent,
|
||||
policy: self.policy,
|
||||
sandbox_id: self.sandbox_id,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
struct FirecrackerReady {
|
||||
client: Arc<GuestClient>,
|
||||
agent: AgentSpec,
|
||||
policy: openshell_core::policy::SandboxPolicy,
|
||||
sandbox_id: String,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl ReadyBoundary for FirecrackerReady {
|
||||
async fn start_agent(self: Box<Self>) -> Result<Box<dyn RunningBoundary>, BackendError> {
|
||||
let response = self
|
||||
.client
|
||||
.call(Request::StartAgent {
|
||||
sandbox_id: self.sandbox_id,
|
||||
spec: AgentSpecWire::from(self.agent),
|
||||
policy: Box::new(SandboxPolicyWire::from(self.policy)),
|
||||
})
|
||||
.await?;
|
||||
let Response::Started { process_id } = response else {
|
||||
return Err(unexpected_response("started", &response));
|
||||
};
|
||||
let process = Arc::new(FirecrackerProcess {
|
||||
client: self.client,
|
||||
process_id,
|
||||
});
|
||||
Ok(Box::new(FirecrackerRunning {
|
||||
process,
|
||||
exec: Arc::new(UnsupportedExec),
|
||||
port_forward: Arc::new(UnsupportedPortForward),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
struct FirecrackerRunning {
|
||||
process: Arc<FirecrackerProcess>,
|
||||
exec: Arc<UnsupportedExec>,
|
||||
port_forward: Arc<UnsupportedPortForward>,
|
||||
}
|
||||
|
||||
impl RunningBoundary for FirecrackerRunning {
|
||||
fn agent(&self) -> Arc<dyn BoundaryProcess> {
|
||||
self.process.clone()
|
||||
}
|
||||
|
||||
fn exec(&self) -> Arc<dyn BoundaryExec> {
|
||||
self.exec.clone()
|
||||
}
|
||||
|
||||
fn port_forward(&self) -> Arc<dyn BoundaryPortForward> {
|
||||
self.port_forward.clone()
|
||||
}
|
||||
}
|
||||
|
||||
struct FirecrackerProcess {
|
||||
client: Arc<GuestClient>,
|
||||
process_id: String,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl BoundaryProcess for FirecrackerProcess {
|
||||
async fn wait(&self) -> Result<BoundaryExitStatus, BackendError> {
|
||||
let response = self
|
||||
.client
|
||||
.call_wait(Request::Wait {
|
||||
process_id: self.process_id.clone(),
|
||||
})
|
||||
.await?;
|
||||
let Response::Exited { status } = response else {
|
||||
return Err(unexpected_response("exited", &response));
|
||||
};
|
||||
Ok(status.into())
|
||||
}
|
||||
|
||||
async fn signal(&self, signal: BoundarySignal) -> Result<(), BackendError> {
|
||||
let response = self
|
||||
.client
|
||||
.call(Request::Signal {
|
||||
process_id: self.process_id.clone(),
|
||||
signal: SignalWire::from(signal),
|
||||
})
|
||||
.await?;
|
||||
expect_response(response, "signaled")
|
||||
}
|
||||
|
||||
async fn terminate(&self) -> Result<(), BackendError> {
|
||||
let response = self
|
||||
.client
|
||||
.call(Request::Terminate {
|
||||
process_id: self.process_id.clone(),
|
||||
})
|
||||
.await?;
|
||||
expect_response(response, "terminated")
|
||||
}
|
||||
}
|
||||
|
||||
struct UnsupportedExec;
|
||||
|
||||
#[async_trait]
|
||||
impl BoundaryExec for UnsupportedExec {
|
||||
async fn exec(&self, _spec: ExecSpec) -> Result<ExecSession, BackendError> {
|
||||
Err(BackendError::Unavailable(
|
||||
"Firecracker exec and PTY transport are not implemented in this prototype".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
struct UnsupportedPortForward;
|
||||
|
||||
#[async_trait]
|
||||
impl BoundaryPortForward for UnsupportedPortForward {
|
||||
async fn connect(&self, _target: LoopbackTarget) -> Result<BoundaryDuplexStream, BackendError> {
|
||||
Err(BackendError::Unavailable(
|
||||
"Firecracker loopback forwarding is not implemented in this prototype".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// The prototype provisions no guest NIC. There can be no workload egress to
|
||||
/// accept, and attempting to consume the source fails the boundary closed.
|
||||
struct NoGuestNetwork;
|
||||
|
||||
#[async_trait]
|
||||
impl NetworkMediationSource for NoGuestNetwork {
|
||||
async fn accept(&self) -> Result<MediatedConnection, BackendError> {
|
||||
Err(BackendError::Unavailable(
|
||||
"Firecracker guest has no NIC; mediated egress is unavailable".to_string(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
struct GuestClient {
|
||||
topology: FirecrackerTopology,
|
||||
next_request_id: AtomicU64,
|
||||
}
|
||||
|
||||
impl GuestClient {
|
||||
fn new(topology: FirecrackerTopology) -> Self {
|
||||
Self {
|
||||
topology,
|
||||
next_request_id: AtomicU64::new(1),
|
||||
}
|
||||
}
|
||||
|
||||
async fn call(&self, request: Request) -> Result<Response, BackendError> {
|
||||
tokio::time::timeout(REQUEST_TIMEOUT, self.exchange(request))
|
||||
.await
|
||||
.map_err(|_| BackendError::Unavailable("guest control request timed out".to_string()))?
|
||||
}
|
||||
|
||||
async fn call_wait(&self, request: Request) -> Result<Response, BackendError> {
|
||||
self.exchange(request).await
|
||||
}
|
||||
|
||||
async fn exchange(&self, request: Request) -> Result<Response, BackendError> {
|
||||
let request_id = self.next_request_id.fetch_add(1, Ordering::Relaxed);
|
||||
let envelope = RequestEnvelope {
|
||||
request_id,
|
||||
boundary_id: self.topology.boundary_id.clone(),
|
||||
bootstrap_token: self.topology.bootstrap_token.clone(),
|
||||
request,
|
||||
};
|
||||
let mut stream = self.connect_vsock().await?;
|
||||
let frame = encode_frame(&envelope)
|
||||
.map_err(|error| BackendError::Process(format!("encode control request: {error}")))?;
|
||||
stream.write_all(&frame).await.map_err(|error| {
|
||||
BackendError::Unavailable(format!("write guest control request: {error}"))
|
||||
})?;
|
||||
let mut header = [0_u8; 4];
|
||||
stream.read_exact(&mut header).await.map_err(|error| {
|
||||
BackendError::Unavailable(format!("read guest control response header: {error}"))
|
||||
})?;
|
||||
let declared = u32::from_be_bytes(header) as usize;
|
||||
if declared > MAX_CONTROL_FRAME_BYTES {
|
||||
return Err(BackendError::Process(format!(
|
||||
"guest control response is too large: {declared} bytes"
|
||||
)));
|
||||
}
|
||||
let mut frame = Vec::with_capacity(4 + declared);
|
||||
frame.extend_from_slice(&header);
|
||||
frame.resize(4 + declared, 0);
|
||||
stream.read_exact(&mut frame[4..]).await.map_err(|error| {
|
||||
BackendError::Unavailable(format!("read guest control response: {error}"))
|
||||
})?;
|
||||
let response: ResponseEnvelope = decode_frame(&frame)
|
||||
.map_err(|error| BackendError::Process(format!("decode control response: {error}")))?;
|
||||
if response.request_id != request_id {
|
||||
return Err(BackendError::Process(format!(
|
||||
"guest response ID {} did not match request ID {request_id}",
|
||||
response.request_id
|
||||
)));
|
||||
}
|
||||
match response.response {
|
||||
Response::Error { kind, message } => Err(guest_error(&kind, message)),
|
||||
response => Ok(response),
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect_vsock(&self) -> Result<UnixStream, BackendError> {
|
||||
let mut stream = UnixStream::connect(&self.topology.vsock_uds_path)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
BackendError::Unavailable(format!("connect to Firecracker vsock UDS: {error}"))
|
||||
})?;
|
||||
stream
|
||||
.write_all(format!("CONNECT {}\n", self.topology.control_port).as_bytes())
|
||||
.await
|
||||
.map_err(|error| {
|
||||
BackendError::Unavailable(format!("write Firecracker vsock handshake: {error}"))
|
||||
})?;
|
||||
let acknowledgment = read_vsock_acknowledgment(&mut stream).await?;
|
||||
validate_vsock_acknowledgment(&acknowledgment)?;
|
||||
Ok(stream)
|
||||
}
|
||||
}
|
||||
|
||||
async fn read_vsock_acknowledgment(stream: &mut UnixStream) -> Result<Vec<u8>, BackendError> {
|
||||
let mut acknowledgment = Vec::with_capacity(24);
|
||||
loop {
|
||||
if acknowledgment.len() == MAX_VSOCK_ACK_BYTES {
|
||||
return Err(BackendError::Process(
|
||||
"Firecracker vsock acknowledgment is too long".to_string(),
|
||||
));
|
||||
}
|
||||
let byte = stream.read_u8().await.map_err(|error| {
|
||||
BackendError::Unavailable(format!("read Firecracker vsock acknowledgment: {error}"))
|
||||
})?;
|
||||
acknowledgment.push(byte);
|
||||
if byte == b'\n' {
|
||||
return Ok(acknowledgment);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_vsock_acknowledgment(acknowledgment: &[u8]) -> Result<(), BackendError> {
|
||||
let acknowledgment = std::str::from_utf8(acknowledgment).map_err(|_| {
|
||||
BackendError::Process("Firecracker vsock acknowledgment is not UTF-8".to_string())
|
||||
})?;
|
||||
let mut fields = acknowledgment
|
||||
.trim_end_matches('\n')
|
||||
.split_ascii_whitespace();
|
||||
let status = fields.next();
|
||||
let assigned_port = fields.next().and_then(|port| port.parse::<u32>().ok());
|
||||
if status != Some("OK") || assigned_port.is_none() || fields.next().is_some() {
|
||||
return Err(BackendError::Process(format!(
|
||||
"invalid Firecracker vsock acknowledgment: {acknowledgment:?}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn expect_response(response: Response, expected: &str) -> Result<(), BackendError> {
|
||||
let matches = matches!(
|
||||
(&response, expected),
|
||||
(Response::Attached, "attached")
|
||||
| (Response::Confirmed, "confirmed")
|
||||
| (Response::Signaled, "signaled")
|
||||
| (Response::Terminated, "terminated")
|
||||
);
|
||||
if matches {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(unexpected_response(expected, &response))
|
||||
}
|
||||
}
|
||||
|
||||
fn unexpected_response(expected: &str, response: &Response) -> BackendError {
|
||||
BackendError::Process(format!(
|
||||
"expected guest response {expected:?}, received {response:?}"
|
||||
))
|
||||
}
|
||||
|
||||
fn guest_error(kind: &str, message: String) -> BackendError {
|
||||
let message = format!("Firecracker guest process leaf: {message}");
|
||||
match kind {
|
||||
"invalid" => BackendError::Descriptor(message),
|
||||
"denied" => BackendError::Denied(message),
|
||||
"unavailable" => BackendError::Unavailable(message),
|
||||
"terminated" => BackendError::Terminated(message),
|
||||
_ => BackendError::Process(message),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::path::PathBuf;
|
||||
|
||||
use openshell_core::policy::{
|
||||
FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
||||
fn sandbox() -> SandboxContext {
|
||||
SandboxContext {
|
||||
sandbox_id: "sandbox-1".to_string(),
|
||||
policy: SandboxPolicy {
|
||||
version: 1,
|
||||
filesystem: FilesystemPolicy::default(),
|
||||
network: NetworkPolicy::default(),
|
||||
landlock: LandlockPolicy::default(),
|
||||
process: ProcessPolicy::default(),
|
||||
},
|
||||
agent: AgentSpec {
|
||||
program: "/bin/true".to_string(),
|
||||
args: Vec::new(),
|
||||
workdir: Some("/sandbox".to_string()),
|
||||
timeout_secs: 5,
|
||||
interactive: false,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_debug_redacts_token() {
|
||||
let topology = FirecrackerTopology {
|
||||
boundary_id: "sandbox-1".to_string(),
|
||||
vsock_uds_path: PathBuf::from("/tmp/vsock.sock"),
|
||||
control_port: 5500,
|
||||
bootstrap_token: "never-log-this-never-log-this".to_string(),
|
||||
};
|
||||
let debug = format!("{topology:?}");
|
||||
assert!(debug.contains("<redacted>"));
|
||||
assert!(!debug.contains("never-log-this"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn topology_must_match_sandbox() {
|
||||
let topology = FirecrackerTopology {
|
||||
boundary_id: "other".to_string(),
|
||||
vsock_uds_path: PathBuf::from("/tmp/vsock.sock"),
|
||||
control_port: 5500,
|
||||
bootstrap_token: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
};
|
||||
assert!(matches!(
|
||||
validate_topology(&topology, &sandbox()),
|
||||
Err(BackendError::Descriptor(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validates_firecracker_acknowledgment() {
|
||||
assert!(validate_vsock_acknowledgment(b"OK 1234\n").is_ok());
|
||||
assert!(validate_vsock_acknowledgment(b"ERR\n").is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,601 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Private guest mode for the Firecracker driver.
|
||||
//!
|
||||
//! This is transport and lifecycle glue, not another supervisor model. When
|
||||
//! the host authorizes `start_agent`, it invokes the existing
|
||||
//! `openshell-supervisor-process` implementation inside the VM.
|
||||
|
||||
#![allow(unsafe_code)]
|
||||
|
||||
use std::ffi::CString;
|
||||
use std::fs::File;
|
||||
use std::io::{self, Read, Write};
|
||||
use std::mem::size_of;
|
||||
use std::os::fd::{AsRawFd, FromRawFd, OwnedFd};
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::AtomicU32;
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use openshell_core::proposals::AgentProposals;
|
||||
use openshell_core::provider_credentials::ProviderCredentialState;
|
||||
use openshell_supervisor_process::boundary_io::BoundaryRuntimeState;
|
||||
use openshell_supervisor_process::process::{
|
||||
ProcessEnforcementMode, ProcessStatus, ResolvedProcessIdentity,
|
||||
};
|
||||
use openshell_supervisor_process::run::{AgentSignaler, spawn_workload};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::protocol::{
|
||||
AgentSpecWire, ExitStatusWire, Request, RequestEnvelope, Response, ResponseEnvelope,
|
||||
SandboxPolicyWire, SignalWire, read_frame, write_frame,
|
||||
};
|
||||
|
||||
pub const DEFAULT_GUEST_CONFIG_PATH: &str = "/etc/openshell/firecracker.json";
|
||||
const DEFAULT_CONTROL_PORT: u32 = 5500;
|
||||
const DEFAULT_AGENT_UID: u32 = 10_001;
|
||||
const DEFAULT_AGENT_GID: u32 = 10_001;
|
||||
const CONTROL_IO_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Driver-private configuration injected into the guest image at provision time.
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
pub struct GuestConfig {
|
||||
pub boundary_id: String,
|
||||
pub bootstrap_token: String,
|
||||
#[serde(default = "default_control_port")]
|
||||
pub control_port: u32,
|
||||
#[serde(default = "default_agent_uid")]
|
||||
pub agent_uid: u32,
|
||||
#[serde(default = "default_agent_gid")]
|
||||
pub agent_gid: u32,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for GuestConfig {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
formatter
|
||||
.debug_struct("GuestConfig")
|
||||
.field("boundary_id", &self.boundary_id)
|
||||
.field("bootstrap_token", &"<redacted>")
|
||||
.field("control_port", &self.control_port)
|
||||
.field("agent_uid", &self.agent_uid)
|
||||
.field("agent_gid", &self.agent_gid)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
const fn default_control_port() -> u32 {
|
||||
DEFAULT_CONTROL_PORT
|
||||
}
|
||||
|
||||
const fn default_agent_uid() -> u32 {
|
||||
DEFAULT_AGENT_UID
|
||||
}
|
||||
|
||||
const fn default_agent_gid() -> u32 {
|
||||
DEFAULT_AGENT_GID
|
||||
}
|
||||
|
||||
pub fn run_guest(config_path: &Path) -> Result<(), String> {
|
||||
if std::process::id() == 1 {
|
||||
prepare_pid1_filesystems()?;
|
||||
}
|
||||
let bytes = std::fs::read(config_path)
|
||||
.map_err(|error| format!("read guest config {}: {error}", config_path.display()))?;
|
||||
let config: GuestConfig = serde_json::from_slice(&bytes)
|
||||
.map_err(|error| format!("decode guest config {}: {error}", config_path.display()))?;
|
||||
validate_config(&config)?;
|
||||
let process_runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|error| format!("create guest process runtime: {error}"))?;
|
||||
let runtime = Arc::new(GuestRuntime::new(
|
||||
config.clone(),
|
||||
process_runtime.handle().clone(),
|
||||
));
|
||||
serve(config.control_port, runtime)
|
||||
}
|
||||
|
||||
fn validate_config(config: &GuestConfig) -> Result<(), String> {
|
||||
if config.boundary_id.is_empty() {
|
||||
return Err("guest boundary ID must not be empty".to_string());
|
||||
}
|
||||
if config.bootstrap_token.len() < 32 {
|
||||
return Err("guest bootstrap token must contain at least 32 bytes".to_string());
|
||||
}
|
||||
if config.control_port == 0 {
|
||||
return Err("guest control port must be nonzero".to_string());
|
||||
}
|
||||
if config.agent_uid == 0 || config.agent_gid == 0 {
|
||||
return Err("guest agent UID and GID must be nonzero".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn prepare_pid1_filesystems() -> Result<(), String> {
|
||||
for path in ["/proc", "/sys", "/dev", "/run", "/tmp", "/sandbox"] {
|
||||
std::fs::create_dir_all(path).map_err(|error| format!("create {path}: {error}"))?;
|
||||
}
|
||||
mount_if_needed("proc", "/proc", "proc")?;
|
||||
mount_if_needed("sysfs", "/sys", "sysfs")?;
|
||||
mount_if_needed("devtmpfs", "/dev", "devtmpfs")?;
|
||||
std::fs::create_dir_all("/dev/pts").map_err(|error| format!("create /dev/pts: {error}"))?;
|
||||
mount_if_needed("devpts", "/dev/pts", "devpts")?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn mount_if_needed(source: &str, target: &str, file_system: &str) -> Result<(), String> {
|
||||
let source = CString::new(source).map_err(|error| error.to_string())?;
|
||||
let target_c = CString::new(target).map_err(|error| error.to_string())?;
|
||||
let file_system = CString::new(file_system).map_err(|error| error.to_string())?;
|
||||
let result = unsafe {
|
||||
libc::mount(
|
||||
source.as_ptr(),
|
||||
target_c.as_ptr(),
|
||||
file_system.as_ptr(),
|
||||
0,
|
||||
std::ptr::null(),
|
||||
)
|
||||
};
|
||||
if result == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
let error = io::Error::last_os_error();
|
||||
if error.raw_os_error() == Some(libc::EBUSY) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("mount {file_system:?} on {target}: {error}"))
|
||||
}
|
||||
}
|
||||
|
||||
fn serve(port: u32, runtime: Arc<GuestRuntime>) -> Result<(), String> {
|
||||
let listener = VsockListener::bind(port)
|
||||
.map_err(|error| format!("bind guest control vsock port {port}: {error}"))?;
|
||||
eprintln!("Firecracker process supervisor leaf listening on vsock port {port}");
|
||||
loop {
|
||||
match listener.accept() {
|
||||
Ok(stream) => {
|
||||
let runtime = runtime.clone();
|
||||
std::thread::spawn(move || {
|
||||
if let Err(error) = serve_one(stream, &runtime) {
|
||||
eprintln!("Firecracker guest control request failed: {error}");
|
||||
}
|
||||
});
|
||||
}
|
||||
Err(error) if error.kind() == io::ErrorKind::Interrupted => {}
|
||||
Err(error) => return Err(format!("accept guest control connection: {error}")),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn serve_one(mut stream: VsockStream, runtime: &GuestRuntime) -> Result<(), String> {
|
||||
stream
|
||||
.set_timeout(CONTROL_IO_TIMEOUT)
|
||||
.map_err(|error| format!("set control timeout: {error}"))?;
|
||||
let request: RequestEnvelope =
|
||||
read_frame(&mut stream).map_err(|error| format!("read control frame: {error}"))?;
|
||||
let response = ResponseEnvelope {
|
||||
request_id: request.request_id,
|
||||
response: runtime.dispatch(request),
|
||||
};
|
||||
write_frame(&mut stream, &response).map_err(|error| format!("write control frame: {error}"))
|
||||
}
|
||||
|
||||
struct GuestRuntime {
|
||||
config: GuestConfig,
|
||||
process_runtime: tokio::runtime::Handle,
|
||||
state: Mutex<RuntimeState>,
|
||||
}
|
||||
|
||||
enum RuntimeState {
|
||||
AwaitingAttach,
|
||||
Bound,
|
||||
Ready,
|
||||
Running(Arc<ManagedProcess>),
|
||||
}
|
||||
|
||||
impl GuestRuntime {
|
||||
fn new(config: GuestConfig, process_runtime: tokio::runtime::Handle) -> Self {
|
||||
Self {
|
||||
config,
|
||||
process_runtime,
|
||||
state: Mutex::new(RuntimeState::AwaitingAttach),
|
||||
}
|
||||
}
|
||||
|
||||
fn dispatch(&self, envelope: RequestEnvelope) -> Response {
|
||||
if !constant_time_eq(
|
||||
envelope.boundary_id.as_bytes(),
|
||||
self.config.boundary_id.as_bytes(),
|
||||
) || !constant_time_eq(
|
||||
envelope.bootstrap_token.as_bytes(),
|
||||
self.config.bootstrap_token.as_bytes(),
|
||||
) {
|
||||
return guest_error("denied", "control authentication failed");
|
||||
}
|
||||
match envelope.request {
|
||||
Request::Attach => self.attach(),
|
||||
Request::Confirm => self.confirm(),
|
||||
Request::StartAgent {
|
||||
sandbox_id,
|
||||
spec,
|
||||
policy,
|
||||
} => self.start_agent(sandbox_id, spec, *policy),
|
||||
Request::Wait { process_id } => self.wait(&process_id),
|
||||
Request::Signal { process_id, signal } => self.signal(&process_id, signal),
|
||||
Request::Terminate { process_id } => self.terminate(&process_id),
|
||||
}
|
||||
}
|
||||
|
||||
fn attach(&self) -> Response {
|
||||
let mut state = lock(&self.state);
|
||||
match *state {
|
||||
RuntimeState::AwaitingAttach => {
|
||||
*state = RuntimeState::Bound;
|
||||
Response::Attached
|
||||
}
|
||||
RuntimeState::Bound => Response::Attached,
|
||||
_ => guest_error("invalid", "boundary has already advanced past attach"),
|
||||
}
|
||||
}
|
||||
|
||||
fn confirm(&self) -> Response {
|
||||
let mut state = lock(&self.state);
|
||||
match *state {
|
||||
RuntimeState::Bound => {
|
||||
*state = RuntimeState::Ready;
|
||||
Response::Confirmed
|
||||
}
|
||||
RuntimeState::Ready => Response::Confirmed,
|
||||
RuntimeState::AwaitingAttach => {
|
||||
guest_error("invalid", "boundary must be attached before confirm")
|
||||
}
|
||||
RuntimeState::Running(_) => {
|
||||
guest_error("invalid", "boundary has already started its agent")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn start_agent(
|
||||
&self,
|
||||
sandbox_id: String,
|
||||
spec: AgentSpecWire,
|
||||
policy: SandboxPolicyWire,
|
||||
) -> Response {
|
||||
let mut state = lock(&self.state);
|
||||
if !matches!(*state, RuntimeState::Ready) {
|
||||
return guest_error("invalid", "boundary must be confirmed before start_agent");
|
||||
}
|
||||
let process = match ManagedProcess::spawn(
|
||||
&self.process_runtime,
|
||||
sandbox_id,
|
||||
spec,
|
||||
policy.into(),
|
||||
ResolvedProcessIdentity::new(Some(self.config.agent_uid), Some(self.config.agent_gid)),
|
||||
) {
|
||||
Ok(process) => Arc::new(process),
|
||||
Err(error) => return guest_error("failed", error),
|
||||
};
|
||||
let process_id = process.process_id();
|
||||
*state = RuntimeState::Running(process);
|
||||
Response::Started { process_id }
|
||||
}
|
||||
|
||||
fn wait(&self, process_id: &str) -> Response {
|
||||
let process = match self.running_process(process_id) {
|
||||
Ok(process) => process,
|
||||
Err(response) => return response,
|
||||
};
|
||||
match process.wait() {
|
||||
Ok(status) => Response::Exited { status },
|
||||
Err(error) => guest_error("failed", error),
|
||||
}
|
||||
}
|
||||
|
||||
fn signal(&self, process_id: &str, signal: SignalWire) -> Response {
|
||||
let process = match self.running_process(process_id) {
|
||||
Ok(process) => process,
|
||||
Err(response) => return response,
|
||||
};
|
||||
match process.signal(signal) {
|
||||
Ok(()) => Response::Signaled,
|
||||
Err(error) => guest_error("terminated", error),
|
||||
}
|
||||
}
|
||||
|
||||
fn terminate(&self, process_id: &str) -> Response {
|
||||
let process = match self.running_process(process_id) {
|
||||
Ok(process) => process,
|
||||
Err(response) => return response,
|
||||
};
|
||||
match process.signal(SignalWire::Kill) {
|
||||
Ok(()) => Response::Terminated,
|
||||
Err(_) if process.has_exited() => Response::Terminated,
|
||||
Err(error) => guest_error("failed", error),
|
||||
}
|
||||
}
|
||||
|
||||
fn running_process(&self, process_id: &str) -> Result<Arc<ManagedProcess>, Response> {
|
||||
let state = lock(&self.state);
|
||||
let RuntimeState::Running(process) = &*state else {
|
||||
return Err(guest_error("invalid", "agent process has not been started"));
|
||||
};
|
||||
if process.process_id() != process_id {
|
||||
return Err(guest_error("invalid", "unknown process ID"));
|
||||
}
|
||||
Ok(process.clone())
|
||||
}
|
||||
}
|
||||
|
||||
type ProcessExit = Result<ExitStatusWire, String>;
|
||||
type SharedProcessExit = Arc<(Mutex<Option<ProcessExit>>, Condvar)>;
|
||||
|
||||
struct ManagedProcess {
|
||||
pid: i32,
|
||||
signaler: AgentSignaler,
|
||||
exit: SharedProcessExit,
|
||||
}
|
||||
|
||||
impl ManagedProcess {
|
||||
fn spawn(
|
||||
runtime: &tokio::runtime::Handle,
|
||||
sandbox_id: String,
|
||||
spec: AgentSpecWire,
|
||||
policy: openshell_core::policy::SandboxPolicy,
|
||||
resolved_identity: ResolvedProcessIdentity,
|
||||
) -> Result<Self, String> {
|
||||
if spec.program.is_empty() {
|
||||
return Err("agent program must not be empty".to_string());
|
||||
}
|
||||
let boundary_runtime = BoundaryRuntimeState::new();
|
||||
let mut spawned = runtime
|
||||
.block_on(spawn_workload(
|
||||
&spec.program,
|
||||
&spec.args,
|
||||
spec.workdir.as_deref(),
|
||||
spec.timeout_secs,
|
||||
spec.interactive,
|
||||
Some(&sandbox_id),
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
&policy,
|
||||
resolved_identity,
|
||||
ProcessEnforcementMode::Full,
|
||||
Arc::new(AtomicU32::new(0)),
|
||||
None,
|
||||
ProviderCredentialState::from_child_env_snapshot(
|
||||
0,
|
||||
std::collections::HashMap::new(),
|
||||
),
|
||||
std::collections::HashMap::new(),
|
||||
None,
|
||||
AgentProposals::default(),
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
Some(boundary_runtime),
|
||||
))
|
||||
.map_err(|error| format!("start process supervisor leaf: {error}"))?;
|
||||
let pid = i32::try_from(spawned.pid())
|
||||
.map_err(|_| "process supervisor PID does not fit i32".to_string())?;
|
||||
let signaler = spawned.signaler();
|
||||
let exit = Arc::new((Mutex::new(None), Condvar::new()));
|
||||
let reaper_exit = exit.clone();
|
||||
runtime.spawn(async move {
|
||||
let result = spawned
|
||||
.wait()
|
||||
.await
|
||||
.map(process_status)
|
||||
.map_err(|error| format!("wait for process supervisor leaf: {error}"));
|
||||
let (state, changed) = &*reaper_exit;
|
||||
*lock(state) = Some(result);
|
||||
changed.notify_all();
|
||||
});
|
||||
Ok(Self {
|
||||
pid,
|
||||
signaler,
|
||||
exit,
|
||||
})
|
||||
}
|
||||
|
||||
fn process_id(&self) -> String {
|
||||
self.pid.to_string()
|
||||
}
|
||||
|
||||
fn wait(&self) -> ProcessExit {
|
||||
let (state, changed) = &*self.exit;
|
||||
let mut exit = lock(state);
|
||||
while exit.is_none() {
|
||||
exit = changed
|
||||
.wait(exit)
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
}
|
||||
exit.as_ref().expect("exit checked above").clone()
|
||||
}
|
||||
|
||||
fn signal(&self, signal: SignalWire) -> Result<(), String> {
|
||||
if self.has_exited() {
|
||||
return Err("agent process has already exited".to_string());
|
||||
}
|
||||
let result = match signal {
|
||||
SignalWire::Term => self.signaler.term(),
|
||||
SignalWire::Kill => self.signaler.kill(),
|
||||
SignalWire::Int => self.signaler.interrupt(),
|
||||
SignalWire::Hup => self.signaler.hangup(),
|
||||
};
|
||||
result.map_err(|error| format!("signal process supervisor group: {error}"))
|
||||
}
|
||||
|
||||
fn has_exited(&self) -> bool {
|
||||
let (state, _) = &*self.exit;
|
||||
lock(state).is_some()
|
||||
}
|
||||
}
|
||||
|
||||
fn lock<T>(mutex: &Mutex<T>) -> std::sync::MutexGuard<'_, T> {
|
||||
mutex
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
}
|
||||
|
||||
fn process_status(status: ProcessStatus) -> ExitStatusWire {
|
||||
status.signal().map_or_else(
|
||||
|| ExitStatusWire::Exited(status.code()),
|
||||
ExitStatusWire::Signaled,
|
||||
)
|
||||
}
|
||||
|
||||
fn guest_error(kind: &str, message: impl Into<String>) -> Response {
|
||||
Response::Error {
|
||||
kind: kind.to_string(),
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
fn constant_time_eq(left: &[u8], right: &[u8]) -> bool {
|
||||
let max_len = left.len().max(right.len());
|
||||
let mut difference = left.len() ^ right.len();
|
||||
for index in 0..max_len {
|
||||
let left_byte = left.get(index).copied().unwrap_or_default();
|
||||
let right_byte = right.get(index).copied().unwrap_or_default();
|
||||
difference |= usize::from(left_byte ^ right_byte);
|
||||
}
|
||||
difference == 0
|
||||
}
|
||||
|
||||
struct VsockListener {
|
||||
fd: OwnedFd,
|
||||
}
|
||||
|
||||
impl VsockListener {
|
||||
fn bind(port: u32) -> io::Result<Self> {
|
||||
let family = libc::sa_family_t::try_from(libc::AF_VSOCK).map_err(|_| {
|
||||
io::Error::new(io::ErrorKind::InvalidInput, "AF_VSOCK exceeds sa_family_t")
|
||||
})?;
|
||||
let address_length =
|
||||
libc::socklen_t::try_from(size_of::<libc::sockaddr_vm>()).map_err(|_| {
|
||||
io::Error::new(io::ErrorKind::InvalidInput, "sockaddr_vm exceeds socklen_t")
|
||||
})?;
|
||||
let raw_fd =
|
||||
unsafe { libc::socket(libc::AF_VSOCK, libc::SOCK_STREAM | libc::SOCK_CLOEXEC, 0) };
|
||||
if raw_fd < 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
let fd = unsafe { OwnedFd::from_raw_fd(raw_fd) };
|
||||
let address = libc::sockaddr_vm {
|
||||
svm_family: family,
|
||||
svm_reserved1: 0,
|
||||
svm_port: port,
|
||||
svm_cid: libc::VMADDR_CID_ANY,
|
||||
svm_zero: [0; 4],
|
||||
};
|
||||
let result = unsafe {
|
||||
libc::bind(
|
||||
fd.as_raw_fd(),
|
||||
(&raw const address).cast::<libc::sockaddr>(),
|
||||
address_length,
|
||||
)
|
||||
};
|
||||
if result < 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
if unsafe { libc::listen(fd.as_raw_fd(), 16) } < 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
Ok(Self { fd })
|
||||
}
|
||||
|
||||
fn accept(&self) -> io::Result<VsockStream> {
|
||||
let raw_fd = unsafe {
|
||||
libc::accept4(
|
||||
self.fd.as_raw_fd(),
|
||||
std::ptr::null_mut(),
|
||||
std::ptr::null_mut(),
|
||||
libc::SOCK_CLOEXEC,
|
||||
)
|
||||
};
|
||||
if raw_fd < 0 {
|
||||
Err(io::Error::last_os_error())
|
||||
} else {
|
||||
Ok(VsockStream {
|
||||
file: unsafe { File::from_raw_fd(raw_fd) },
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct VsockStream {
|
||||
file: File,
|
||||
}
|
||||
|
||||
impl VsockStream {
|
||||
fn set_timeout(&self, timeout: Duration) -> io::Result<()> {
|
||||
let option_length =
|
||||
libc::socklen_t::try_from(size_of::<libc::timeval>()).map_err(|_| {
|
||||
io::Error::new(io::ErrorKind::InvalidInput, "timeval exceeds socklen_t")
|
||||
})?;
|
||||
let timeout = libc::timeval {
|
||||
tv_sec: timeout.as_secs().try_into().unwrap_or(libc::time_t::MAX),
|
||||
tv_usec: timeout.subsec_micros().into(),
|
||||
};
|
||||
for option in [libc::SO_RCVTIMEO, libc::SO_SNDTIMEO] {
|
||||
let result = unsafe {
|
||||
libc::setsockopt(
|
||||
self.file.as_raw_fd(),
|
||||
libc::SOL_SOCKET,
|
||||
option,
|
||||
(&raw const timeout).cast(),
|
||||
option_length,
|
||||
)
|
||||
};
|
||||
if result < 0 {
|
||||
return Err(io::Error::last_os_error());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Read for VsockStream {
|
||||
fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
|
||||
self.file.read(buffer)
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for VsockStream {
|
||||
fn write(&mut self, buffer: &[u8]) -> io::Result<usize> {
|
||||
self.file.write(buffer)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.file.flush()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn guest_config_debug_redacts_token() {
|
||||
let config = GuestConfig {
|
||||
boundary_id: "sandbox-1".to_string(),
|
||||
bootstrap_token: "never-log-this-never-log-this".to_string(),
|
||||
control_port: 5500,
|
||||
agent_uid: DEFAULT_AGENT_UID,
|
||||
agent_gid: DEFAULT_AGENT_GID,
|
||||
};
|
||||
let debug = format!("{config:?}");
|
||||
assert!(debug.contains("<redacted>"));
|
||||
assert!(!debug.contains("never-log-this"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn constant_time_comparison_checks_length_and_content() {
|
||||
assert!(constant_time_eq(b"same", b"same"));
|
||||
assert!(!constant_time_eq(b"same", b"different"));
|
||||
assert!(!constant_time_eq(b"same", b"sam"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Experimental Firecracker Isolation Backend.
|
||||
//!
|
||||
//! The logical supervisor and RFC 0012 lifecycle live on the host. A private
|
||||
//! guest mode of the same driver binary only transports lifecycle operations
|
||||
//! and invokes the existing `openshell-supervisor-process` implementation.
|
||||
|
||||
pub mod backend;
|
||||
pub mod guest;
|
||||
mod protocol;
|
||||
pub mod runtime;
|
||||
|
||||
pub use backend::{BACKEND_NAME, FirecrackerHostBackend, FirecrackerTopology};
|
||||
pub use guest::{DEFAULT_GUEST_CONFIG_PATH, GuestConfig, run_guest};
|
||||
pub use runtime::{FirecrackerLaunchConfig, FirecrackerVm};
|
||||
@@ -0,0 +1,242 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use clap::{Args, Parser, Subcommand};
|
||||
use openshell_core::policy::{
|
||||
FilesystemPolicy, LandlockPolicy, NetworkPolicy, ProcessPolicy, SandboxPolicy,
|
||||
};
|
||||
use openshell_driver_firecracker::{
|
||||
BACKEND_NAME, DEFAULT_GUEST_CONFIG_PATH, FirecrackerHostBackend, FirecrackerLaunchConfig,
|
||||
FirecrackerTopology, FirecrackerVm, run_guest,
|
||||
};
|
||||
use openshell_isolation::AgentSpec;
|
||||
use openshell_isolation::contract::{
|
||||
BackendRegistry, BoundaryExitStatus, INTERFACE_VERSION, SandboxContext, TopologyDescriptor,
|
||||
};
|
||||
|
||||
#[derive(Debug, Parser)]
|
||||
#[command(about = "Experimental OpenShell Firecracker isolation driver")]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Option<Command>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Subcommand)]
|
||||
enum Command {
|
||||
/// Boot and wait for one Firecracker VM.
|
||||
Launch(LaunchArgs),
|
||||
/// Drive the RFC 0012 lifecycle from the host.
|
||||
Supervise(SuperviseArgs),
|
||||
/// Run the private process-supervisor leaf transport inside the guest.
|
||||
Guest(GuestArgs),
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct LaunchArgs {
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_FIRECRACKER_BINARY",
|
||||
default_value = "firecracker"
|
||||
)]
|
||||
firecracker_binary: PathBuf,
|
||||
#[arg(long)]
|
||||
kernel_image: PathBuf,
|
||||
#[arg(long)]
|
||||
root_disk: PathBuf,
|
||||
#[arg(long)]
|
||||
run_dir: PathBuf,
|
||||
#[arg(long)]
|
||||
console_output: PathBuf,
|
||||
#[arg(
|
||||
long,
|
||||
default_value = "/opt/openshell/bin/openshell-driver-firecracker"
|
||||
)]
|
||||
guest_init: String,
|
||||
#[arg(long, default_value_t = 2)]
|
||||
vcpus: u8,
|
||||
#[arg(long, default_value_t = 512)]
|
||||
mem_mib: u32,
|
||||
#[arg(long)]
|
||||
vsock_cid: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct SuperviseArgs {
|
||||
#[arg(long)]
|
||||
boundary_id: String,
|
||||
#[arg(long)]
|
||||
vsock_uds_path: PathBuf,
|
||||
#[arg(long, default_value_t = 5500)]
|
||||
vsock_port: u32,
|
||||
#[arg(long)]
|
||||
bootstrap_token_file: PathBuf,
|
||||
#[arg(long, default_value = "/sandbox")]
|
||||
workdir: String,
|
||||
#[arg(long, default_value_t = 300)]
|
||||
timeout_seconds: u64,
|
||||
#[arg(required = true, trailing_var_arg = true)]
|
||||
command: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Args)]
|
||||
struct GuestArgs {
|
||||
#[arg(long, default_value = DEFAULT_GUEST_CONFIG_PATH)]
|
||||
config: PathBuf,
|
||||
}
|
||||
|
||||
fn main() {
|
||||
if let Err(error) = run() {
|
||||
eprintln!("Firecracker driver failed: {error}");
|
||||
std::process::exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
fn run() -> Result<(), String> {
|
||||
let cli = Cli::parse();
|
||||
let command = match cli.command {
|
||||
Some(command) => command,
|
||||
None if std::process::id() == 1 => Command::Guest(GuestArgs {
|
||||
config: PathBuf::from(DEFAULT_GUEST_CONFIG_PATH),
|
||||
}),
|
||||
None => return Err("a subcommand is required outside a Firecracker guest".to_string()),
|
||||
};
|
||||
match command {
|
||||
Command::Guest(args) => run_guest(&args.config),
|
||||
Command::Launch(args) => launch(args),
|
||||
Command::Supervise(args) => {
|
||||
let runtime = tokio::runtime::Builder::new_multi_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.map_err(|error| format!("create host runtime: {error}"))?;
|
||||
runtime.block_on(supervise(args))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn launch(args: LaunchArgs) -> Result<(), String> {
|
||||
let config = FirecrackerLaunchConfig {
|
||||
firecracker_binary: resolve_executable(&args.firecracker_binary)?,
|
||||
kernel_image: args.kernel_image,
|
||||
root_disk: args.root_disk,
|
||||
run_dir: args.run_dir,
|
||||
console_output: args.console_output,
|
||||
guest_init: args.guest_init,
|
||||
vcpus: args.vcpus,
|
||||
mem_mib: args.mem_mib,
|
||||
vsock_cid: args.vsock_cid,
|
||||
};
|
||||
let mut vm = FirecrackerVm::launch(&config)?;
|
||||
let status = vm.wait()?;
|
||||
if status.success() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!("Firecracker exited with status {status}"))
|
||||
}
|
||||
}
|
||||
|
||||
async fn supervise(args: SuperviseArgs) -> Result<(), String> {
|
||||
let bootstrap_token = std::fs::read_to_string(&args.bootstrap_token_file)
|
||||
.map_err(|error| {
|
||||
format!(
|
||||
"read bootstrap token file {}: {error}",
|
||||
args.bootstrap_token_file.display()
|
||||
)
|
||||
})?
|
||||
.trim()
|
||||
.to_string();
|
||||
let topology = FirecrackerTopology {
|
||||
boundary_id: args.boundary_id.clone(),
|
||||
vsock_uds_path: args.vsock_uds_path,
|
||||
control_port: args.vsock_port,
|
||||
bootstrap_token,
|
||||
};
|
||||
let descriptor = TopologyDescriptor {
|
||||
version: INTERFACE_VERSION,
|
||||
backend_name: BACKEND_NAME.to_string(),
|
||||
payload: topology.encode().map_err(|error| error.to_string())?,
|
||||
};
|
||||
let mut registry = BackendRegistry::new();
|
||||
registry
|
||||
.register(Arc::new(FirecrackerHostBackend))
|
||||
.map_err(|error| error.to_string())?;
|
||||
let (backend, verified) = registry
|
||||
.resolve(descriptor, BACKEND_NAME)
|
||||
.map_err(|error| error.to_string())?;
|
||||
let (program, command_args) = args
|
||||
.command
|
||||
.split_first()
|
||||
.ok_or_else(|| "agent command must not be empty".to_string())?;
|
||||
let sandbox = SandboxContext {
|
||||
sandbox_id: args.boundary_id,
|
||||
policy: restrictive_host_policy(),
|
||||
agent: AgentSpec {
|
||||
program: program.clone(),
|
||||
args: command_args.to_vec(),
|
||||
workdir: Some(args.workdir),
|
||||
timeout_secs: args.timeout_seconds,
|
||||
interactive: false,
|
||||
},
|
||||
};
|
||||
let bound = backend
|
||||
.attach(verified, sandbox)
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let ready = bound.confirm().await.map_err(|error| error.to_string())?;
|
||||
let running = ready
|
||||
.start_agent()
|
||||
.await
|
||||
.map_err(|error| error.to_string())?;
|
||||
let process = running.agent();
|
||||
let wait = process.wait();
|
||||
tokio::pin!(wait);
|
||||
let status = tokio::select! {
|
||||
result = &mut wait => result.map_err(|error| error.to_string())?,
|
||||
signal = tokio::signal::ctrl_c() => {
|
||||
signal.map_err(|error| format!("wait for interrupt: {error}"))?;
|
||||
process.terminate().await.map_err(|error| error.to_string())?;
|
||||
process.wait().await.map_err(|error| error.to_string())?
|
||||
}
|
||||
};
|
||||
println!("agent exited: {status:?}");
|
||||
match status {
|
||||
BoundaryExitStatus::Exited(0) => Ok(()),
|
||||
_ => Err(format!("agent exited unsuccessfully: {status:?}")),
|
||||
}
|
||||
}
|
||||
|
||||
fn restrictive_host_policy() -> SandboxPolicy {
|
||||
SandboxPolicy {
|
||||
version: 1,
|
||||
filesystem: FilesystemPolicy {
|
||||
read_only: ["/usr", "/lib", "/proc", "/dev/urandom", "/etc", "/var/log"]
|
||||
.into_iter()
|
||||
.map(PathBuf::from)
|
||||
.collect(),
|
||||
read_write: ["/sandbox", "/tmp", "/dev/null"]
|
||||
.into_iter()
|
||||
.map(PathBuf::from)
|
||||
.collect(),
|
||||
include_workdir: true,
|
||||
},
|
||||
network: NetworkPolicy::default(),
|
||||
landlock: LandlockPolicy::default(),
|
||||
process: ProcessPolicy::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn resolve_executable(path: &Path) -> Result<PathBuf, String> {
|
||||
if path.components().count() > 1 || path.is_absolute() {
|
||||
return path
|
||||
.is_file()
|
||||
.then(|| path.to_path_buf())
|
||||
.ok_or_else(|| format!("executable not found: {}", path.display()));
|
||||
}
|
||||
let search_path = std::env::var_os("PATH").unwrap_or_default();
|
||||
std::env::split_paths(&search_path)
|
||||
.map(|directory| directory.join(path))
|
||||
.find(|candidate| candidate.is_file())
|
||||
.ok_or_else(|| format!("executable not found on PATH: {}", path.display()))
|
||||
}
|
||||
@@ -0,0 +1,361 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Driver-private, length-delimited JSON protocol carried over virtio-vsock.
|
||||
|
||||
use std::fmt;
|
||||
use std::io::{self, Read, Write};
|
||||
|
||||
use openshell_core::policy::{
|
||||
FilesystemPolicy, LandlockCompatibility, LandlockPolicy, NetworkMode, NetworkPolicy,
|
||||
ProcessPolicy, ProxyPolicy, SandboxPolicy,
|
||||
};
|
||||
use openshell_isolation::AgentSpec;
|
||||
use openshell_isolation::contract::{BoundaryExitStatus, BoundarySignal};
|
||||
use serde::de::DeserializeOwned;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
pub const MAX_CONTROL_FRAME_BYTES: usize = 1024 * 1024;
|
||||
|
||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RequestEnvelope {
|
||||
pub request_id: u64,
|
||||
pub boundary_id: String,
|
||||
pub bootstrap_token: String,
|
||||
pub request: Request,
|
||||
}
|
||||
|
||||
impl fmt::Debug for RequestEnvelope {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
formatter
|
||||
.debug_struct("RequestEnvelope")
|
||||
.field("request_id", &self.request_id)
|
||||
.field("boundary_id", &self.boundary_id)
|
||||
.field("bootstrap_token", &"<redacted>")
|
||||
.field("request", &self.request)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "operation", rename_all = "snake_case")]
|
||||
pub enum Request {
|
||||
Attach,
|
||||
Confirm,
|
||||
StartAgent {
|
||||
sandbox_id: String,
|
||||
spec: AgentSpecWire,
|
||||
policy: Box<SandboxPolicyWire>,
|
||||
},
|
||||
Wait {
|
||||
process_id: String,
|
||||
},
|
||||
Signal {
|
||||
process_id: String,
|
||||
signal: SignalWire,
|
||||
},
|
||||
Terminate {
|
||||
process_id: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct ResponseEnvelope {
|
||||
pub request_id: u64,
|
||||
pub response: Response,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "result", rename_all = "snake_case")]
|
||||
pub enum Response {
|
||||
Attached,
|
||||
Confirmed,
|
||||
Started { process_id: String },
|
||||
Exited { status: ExitStatusWire },
|
||||
Signaled,
|
||||
Terminated,
|
||||
Error { kind: String, message: String },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AgentSpecWire {
|
||||
pub program: String,
|
||||
pub args: Vec<String>,
|
||||
pub workdir: Option<String>,
|
||||
pub timeout_secs: u64,
|
||||
pub interactive: bool,
|
||||
}
|
||||
|
||||
impl From<AgentSpec> for AgentSpecWire {
|
||||
fn from(spec: AgentSpec) -> Self {
|
||||
Self {
|
||||
program: spec.program,
|
||||
args: spec.args,
|
||||
workdir: spec.workdir,
|
||||
timeout_secs: spec.timeout_secs,
|
||||
interactive: spec.interactive,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct SandboxPolicyWire {
|
||||
pub version: u32,
|
||||
pub read_only: Vec<std::path::PathBuf>,
|
||||
pub read_write: Vec<std::path::PathBuf>,
|
||||
pub include_workdir: bool,
|
||||
pub network: NetworkModeWire,
|
||||
pub proxy_addr: Option<std::net::SocketAddr>,
|
||||
pub landlock: LandlockCompatibilityWire,
|
||||
pub run_as_user: Option<String>,
|
||||
pub run_as_group: Option<String>,
|
||||
}
|
||||
|
||||
impl From<SandboxPolicy> for SandboxPolicyWire {
|
||||
fn from(policy: SandboxPolicy) -> Self {
|
||||
Self {
|
||||
version: policy.version,
|
||||
read_only: policy.filesystem.read_only,
|
||||
read_write: policy.filesystem.read_write,
|
||||
include_workdir: policy.filesystem.include_workdir,
|
||||
network: NetworkModeWire::from(policy.network.mode),
|
||||
proxy_addr: policy.network.proxy.and_then(|proxy| proxy.http_addr),
|
||||
landlock: LandlockCompatibilityWire::from(policy.landlock.compatibility),
|
||||
run_as_user: policy.process.run_as_user,
|
||||
run_as_group: policy.process.run_as_group,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SandboxPolicyWire> for SandboxPolicy {
|
||||
fn from(policy: SandboxPolicyWire) -> Self {
|
||||
let proxy = matches!(policy.network, NetworkModeWire::Proxy).then_some(ProxyPolicy {
|
||||
http_addr: policy.proxy_addr,
|
||||
});
|
||||
Self {
|
||||
version: policy.version,
|
||||
filesystem: FilesystemPolicy {
|
||||
read_only: policy.read_only,
|
||||
read_write: policy.read_write,
|
||||
include_workdir: policy.include_workdir,
|
||||
},
|
||||
network: NetworkPolicy {
|
||||
mode: policy.network.into(),
|
||||
proxy,
|
||||
},
|
||||
landlock: LandlockPolicy {
|
||||
compatibility: policy.landlock.into(),
|
||||
},
|
||||
process: ProcessPolicy {
|
||||
run_as_user: policy.run_as_user,
|
||||
run_as_group: policy.run_as_group,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NetworkModeWire {
|
||||
Block,
|
||||
Proxy,
|
||||
Allow,
|
||||
}
|
||||
|
||||
impl From<NetworkMode> for NetworkModeWire {
|
||||
fn from(mode: NetworkMode) -> Self {
|
||||
match mode {
|
||||
NetworkMode::Block => Self::Block,
|
||||
NetworkMode::Proxy => Self::Proxy,
|
||||
NetworkMode::Allow => Self::Allow,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<NetworkModeWire> for NetworkMode {
|
||||
fn from(mode: NetworkModeWire) -> Self {
|
||||
match mode {
|
||||
NetworkModeWire::Block => Self::Block,
|
||||
NetworkModeWire::Proxy => Self::Proxy,
|
||||
NetworkModeWire::Allow => Self::Allow,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum LandlockCompatibilityWire {
|
||||
BestEffort,
|
||||
HardRequirement,
|
||||
}
|
||||
|
||||
impl From<LandlockCompatibility> for LandlockCompatibilityWire {
|
||||
fn from(compatibility: LandlockCompatibility) -> Self {
|
||||
match compatibility {
|
||||
LandlockCompatibility::BestEffort => Self::BestEffort,
|
||||
LandlockCompatibility::HardRequirement => Self::HardRequirement,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<LandlockCompatibilityWire> for LandlockCompatibility {
|
||||
fn from(compatibility: LandlockCompatibilityWire) -> Self {
|
||||
match compatibility {
|
||||
LandlockCompatibilityWire::BestEffort => Self::BestEffort,
|
||||
LandlockCompatibilityWire::HardRequirement => Self::HardRequirement,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SignalWire {
|
||||
Term,
|
||||
Kill,
|
||||
Int,
|
||||
Hup,
|
||||
}
|
||||
|
||||
impl From<BoundarySignal> for SignalWire {
|
||||
fn from(signal: BoundarySignal) -> Self {
|
||||
match signal {
|
||||
BoundarySignal::Term => Self::Term,
|
||||
BoundarySignal::Kill => Self::Kill,
|
||||
BoundarySignal::Int => Self::Int,
|
||||
BoundarySignal::Hup => Self::Hup,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(tag = "kind", content = "value", rename_all = "snake_case")]
|
||||
pub enum ExitStatusWire {
|
||||
Exited(i32),
|
||||
Signaled(i32),
|
||||
}
|
||||
|
||||
impl From<ExitStatusWire> for BoundaryExitStatus {
|
||||
fn from(status: ExitStatusWire) -> Self {
|
||||
match status {
|
||||
ExitStatusWire::Exited(code) => Self::Exited(code),
|
||||
ExitStatusWire::Signaled(signal) => Self::Signaled(signal),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encode_frame<T: Serialize>(message: &T) -> Result<Vec<u8>, FrameError> {
|
||||
let payload = serde_json::to_vec(message).map_err(FrameError::Serialize)?;
|
||||
if payload.len() > MAX_CONTROL_FRAME_BYTES {
|
||||
return Err(FrameError::TooLarge(payload.len()));
|
||||
}
|
||||
let length = u32::try_from(payload.len()).map_err(|_| FrameError::TooLarge(payload.len()))?;
|
||||
let mut frame = Vec::with_capacity(4 + payload.len());
|
||||
frame.extend_from_slice(&length.to_be_bytes());
|
||||
frame.extend_from_slice(&payload);
|
||||
Ok(frame)
|
||||
}
|
||||
|
||||
pub fn decode_frame<T: DeserializeOwned>(frame: &[u8]) -> Result<T, FrameError> {
|
||||
let header: [u8; 4] = frame
|
||||
.get(..4)
|
||||
.ok_or(FrameError::Truncated)?
|
||||
.try_into()
|
||||
.map_err(|_| FrameError::Truncated)?;
|
||||
let declared = u32::from_be_bytes(header) as usize;
|
||||
if declared > MAX_CONTROL_FRAME_BYTES {
|
||||
return Err(FrameError::TooLarge(declared));
|
||||
}
|
||||
let payload = frame.get(4..).ok_or(FrameError::Truncated)?;
|
||||
if payload.len() != declared {
|
||||
return Err(FrameError::LengthMismatch {
|
||||
declared,
|
||||
actual: payload.len(),
|
||||
});
|
||||
}
|
||||
serde_json::from_slice(payload).map_err(FrameError::Deserialize)
|
||||
}
|
||||
|
||||
pub fn read_frame<T: DeserializeOwned>(reader: &mut impl Read) -> Result<T, FrameError> {
|
||||
let mut header = [0_u8; 4];
|
||||
reader.read_exact(&mut header)?;
|
||||
let declared = u32::from_be_bytes(header) as usize;
|
||||
if declared > MAX_CONTROL_FRAME_BYTES {
|
||||
return Err(FrameError::TooLarge(declared));
|
||||
}
|
||||
let mut frame = Vec::with_capacity(4 + declared);
|
||||
frame.extend_from_slice(&header);
|
||||
frame.resize(4 + declared, 0);
|
||||
reader.read_exact(&mut frame[4..])?;
|
||||
decode_frame(&frame)
|
||||
}
|
||||
|
||||
pub fn write_frame<T: Serialize>(writer: &mut impl Write, message: &T) -> Result<(), FrameError> {
|
||||
let frame = encode_frame(message)?;
|
||||
writer.write_all(&frame)?;
|
||||
writer.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum FrameError {
|
||||
#[error("control frame is truncated")]
|
||||
Truncated,
|
||||
#[error("control frame is too large: {0} bytes")]
|
||||
TooLarge(usize),
|
||||
#[error("control frame declared {declared} bytes but contained {actual}")]
|
||||
LengthMismatch { declared: usize, actual: usize },
|
||||
#[error("serialize control frame: {0}")]
|
||||
Serialize(serde_json::Error),
|
||||
#[error("deserialize control frame: {0}")]
|
||||
Deserialize(serde_json::Error),
|
||||
#[error("read or write control frame: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn request_round_trips_and_redacts_token() {
|
||||
let request = RequestEnvelope {
|
||||
request_id: 7,
|
||||
boundary_id: "sandbox-1".to_string(),
|
||||
bootstrap_token: "never-log-this".to_string(),
|
||||
request: Request::StartAgent {
|
||||
sandbox_id: "sandbox-1".to_string(),
|
||||
spec: AgentSpecWire {
|
||||
program: "/bin/true".to_string(),
|
||||
args: Vec::new(),
|
||||
workdir: Some("/sandbox".to_string()),
|
||||
timeout_secs: 5,
|
||||
interactive: false,
|
||||
},
|
||||
policy: Box::new(SandboxPolicyWire::from(SandboxPolicy {
|
||||
version: 1,
|
||||
filesystem: FilesystemPolicy::default(),
|
||||
network: NetworkPolicy::default(),
|
||||
landlock: LandlockPolicy::default(),
|
||||
process: ProcessPolicy::default(),
|
||||
})),
|
||||
},
|
||||
};
|
||||
let frame = encode_frame(&request).expect("encode request");
|
||||
let decoded: RequestEnvelope = decode_frame(&frame).expect("decode request");
|
||||
assert_eq!(decoded, request);
|
||||
let debug = format!("{request:?}");
|
||||
assert!(debug.contains("<redacted>"));
|
||||
assert!(!debug.contains("never-log-this"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_declared_oversize() {
|
||||
let oversized = u32::try_from(MAX_CONTROL_FRAME_BYTES + 1).expect("test size fits u32");
|
||||
let mut frame = Vec::from(oversized.to_be_bytes());
|
||||
frame.extend_from_slice(b"{}");
|
||||
assert!(matches!(
|
||||
decode_frame::<RequestEnvelope>(&frame),
|
||||
Err(FrameError::TooLarge(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,401 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Small standalone Firecracker launcher owned by this driver crate.
|
||||
|
||||
#![allow(unsafe_code)]
|
||||
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::os::unix::process::CommandExt as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, ExitStatus, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use nix::sys::signal::Signal;
|
||||
use serde_json::{Value, json};
|
||||
|
||||
const API_START_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const API_IO_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
const MAX_API_RESPONSE_BYTES: usize = 1024 * 1024;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct FirecrackerLaunchConfig {
|
||||
pub firecracker_binary: PathBuf,
|
||||
pub kernel_image: PathBuf,
|
||||
pub root_disk: PathBuf,
|
||||
pub run_dir: PathBuf,
|
||||
pub console_output: PathBuf,
|
||||
pub guest_init: String,
|
||||
pub vcpus: u8,
|
||||
pub mem_mib: u32,
|
||||
pub vsock_cid: u32,
|
||||
}
|
||||
|
||||
/// A configured Firecracker child. Dropping it terminates the VMM.
|
||||
pub struct FirecrackerVm {
|
||||
child: Child,
|
||||
api_socket: PathBuf,
|
||||
vsock_socket: PathBuf,
|
||||
}
|
||||
|
||||
impl FirecrackerVm {
|
||||
pub fn launch(config: &FirecrackerLaunchConfig) -> Result<Self, String> {
|
||||
validate_config(config)?;
|
||||
check_kvm_access()?;
|
||||
std::fs::create_dir_all(&config.run_dir).map_err(|error| {
|
||||
format!(
|
||||
"create Firecracker run dir {}: {error}",
|
||||
config.run_dir.display()
|
||||
)
|
||||
})?;
|
||||
let api_socket = config.run_dir.join("firecracker-api.sock");
|
||||
let vsock_socket = config.run_dir.join("firecracker-vsock.sock");
|
||||
remove_stale_socket(&api_socket)?;
|
||||
remove_stale_socket(&vsock_socket)?;
|
||||
|
||||
let console = File::create(&config.console_output).map_err(|error| {
|
||||
format!(
|
||||
"create Firecracker console log {}: {error}",
|
||||
config.console_output.display()
|
||||
)
|
||||
})?;
|
||||
let stderr = console
|
||||
.try_clone()
|
||||
.map_err(|error| format!("clone Firecracker console log: {error}"))?;
|
||||
let mut command = Command::new(&config.firecracker_binary);
|
||||
command
|
||||
.arg("--api-sock")
|
||||
.arg(&api_socket)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::from(console))
|
||||
.stderr(Stdio::from(stderr));
|
||||
unsafe {
|
||||
command.pre_exec(|| {
|
||||
nix::sys::prctl::set_pdeathsig(Signal::SIGKILL)
|
||||
.map_err(|error| io::Error::other(format!("pdeathsig: {error}")))
|
||||
});
|
||||
}
|
||||
let mut child = command
|
||||
.spawn()
|
||||
.map_err(|error| format!("start Firecracker: {error}"))?;
|
||||
if let Err(error) = configure(&mut child, &api_socket, &vsock_socket, config) {
|
||||
terminate_child(&mut child);
|
||||
return Err(error);
|
||||
}
|
||||
Ok(Self {
|
||||
child,
|
||||
api_socket,
|
||||
vsock_socket,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn vsock_uds_path(&self) -> &Path {
|
||||
&self.vsock_socket
|
||||
}
|
||||
|
||||
pub fn wait(&mut self) -> Result<ExitStatus, String> {
|
||||
self.child
|
||||
.wait()
|
||||
.map_err(|error| format!("wait for Firecracker: {error}"))
|
||||
}
|
||||
|
||||
pub fn terminate(&mut self) -> Result<(), String> {
|
||||
terminate_child(&mut self.child);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FirecrackerVm {
|
||||
fn drop(&mut self) {
|
||||
if self.child.try_wait().ok().flatten().is_none() {
|
||||
terminate_child(&mut self.child);
|
||||
}
|
||||
let _ = std::fs::remove_file(&self.api_socket);
|
||||
let _ = std::fs::remove_file(&self.vsock_socket);
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_config(config: &FirecrackerLaunchConfig) -> Result<(), String> {
|
||||
for (label, path) in [
|
||||
("Firecracker binary", &config.firecracker_binary),
|
||||
("kernel image", &config.kernel_image),
|
||||
("root disk", &config.root_disk),
|
||||
] {
|
||||
if !path.is_file() {
|
||||
return Err(format!("{label} not found: {}", path.display()));
|
||||
}
|
||||
}
|
||||
if config.vcpus == 0 {
|
||||
return Err("Firecracker vCPU count must be nonzero".to_string());
|
||||
}
|
||||
if config.mem_mib < 128 {
|
||||
return Err("Firecracker memory must be at least 128 MiB".to_string());
|
||||
}
|
||||
if config.vsock_cid < 3 {
|
||||
return Err("Firecracker guest CID must be at least 3".to_string());
|
||||
}
|
||||
if !config.guest_init.starts_with('/') {
|
||||
return Err("Firecracker guest init path must be absolute".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_kvm_access() -> Result<(), String> {
|
||||
OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.open("/dev/kvm")
|
||||
.map(|_| ())
|
||||
.map_err(|error| {
|
||||
format!(
|
||||
"open /dev/kvm read/write: {error}; start a login session whose supplementary groups include kvm"
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn configure(
|
||||
child: &mut Child,
|
||||
api_socket: &Path,
|
||||
vsock_socket: &Path,
|
||||
config: &FirecrackerLaunchConfig,
|
||||
) -> Result<(), String> {
|
||||
wait_for_api_socket(child, api_socket)?;
|
||||
for request in configuration_requests(config, vsock_socket) {
|
||||
put_json(api_socket, request.path, &request.body)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct ApiRequest {
|
||||
path: &'static str,
|
||||
body: Value,
|
||||
}
|
||||
|
||||
fn configuration_requests(
|
||||
config: &FirecrackerLaunchConfig,
|
||||
vsock_socket: &Path,
|
||||
) -> Vec<ApiRequest> {
|
||||
vec![
|
||||
ApiRequest {
|
||||
path: "/machine-config",
|
||||
body: json!({
|
||||
"vcpu_count": config.vcpus,
|
||||
"mem_size_mib": config.mem_mib,
|
||||
"smt": false
|
||||
}),
|
||||
},
|
||||
ApiRequest {
|
||||
path: "/boot-source",
|
||||
body: json!({
|
||||
"kernel_image_path": config.kernel_image,
|
||||
"boot_args": kernel_command_line(&config.guest_init)
|
||||
}),
|
||||
},
|
||||
ApiRequest {
|
||||
path: "/drives/rootfs",
|
||||
body: json!({
|
||||
"drive_id": "rootfs",
|
||||
"path_on_host": config.root_disk,
|
||||
"is_root_device": true,
|
||||
"is_read_only": false
|
||||
}),
|
||||
},
|
||||
ApiRequest {
|
||||
path: "/vsock",
|
||||
body: json!({
|
||||
"guest_cid": config.vsock_cid,
|
||||
"uds_path": vsock_socket
|
||||
}),
|
||||
},
|
||||
ApiRequest {
|
||||
path: "/actions",
|
||||
body: json!({ "action_type": "InstanceStart" }),
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn kernel_command_line(init: &str) -> String {
|
||||
format!("console=ttyS0 reboot=k panic=1 pci=off root=/dev/vda rw init={init}")
|
||||
}
|
||||
|
||||
fn wait_for_api_socket(child: &mut Child, socket: &Path) -> Result<(), String> {
|
||||
let deadline = Instant::now() + API_START_TIMEOUT;
|
||||
loop {
|
||||
if UnixStream::connect(socket).is_ok() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(status) = child
|
||||
.try_wait()
|
||||
.map_err(|error| format!("check Firecracker process: {error}"))?
|
||||
{
|
||||
return Err(format!(
|
||||
"Firecracker exited before its API socket was ready: {status}"
|
||||
));
|
||||
}
|
||||
if Instant::now() >= deadline {
|
||||
return Err(format!(
|
||||
"timed out waiting for Firecracker API socket {}",
|
||||
socket.display()
|
||||
));
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
}
|
||||
|
||||
fn put_json(socket: &Path, path: &str, body: &Value) -> Result<(), String> {
|
||||
let body = serde_json::to_vec(body).map_err(|error| format!("encode {path}: {error}"))?;
|
||||
let mut stream = UnixStream::connect(socket)
|
||||
.map_err(|error| format!("connect to Firecracker API {}: {error}", socket.display()))?;
|
||||
stream
|
||||
.set_read_timeout(Some(API_IO_TIMEOUT))
|
||||
.map_err(|error| format!("set API read timeout: {error}"))?;
|
||||
stream
|
||||
.set_write_timeout(Some(API_IO_TIMEOUT))
|
||||
.map_err(|error| format!("set API write timeout: {error}"))?;
|
||||
write!(
|
||||
stream,
|
||||
"PUT {path} HTTP/1.1\r\nHost: localhost\r\nAccept: application/json\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
)
|
||||
.and_then(|()| stream.write_all(&body))
|
||||
.map_err(|error| format!("write Firecracker API request {path}: {error}"))?;
|
||||
let response = read_http_response(&mut stream, path)?;
|
||||
check_http_status(path, &response)
|
||||
}
|
||||
|
||||
fn read_http_response(stream: &mut UnixStream, path: &str) -> Result<Vec<u8>, String> {
|
||||
let mut response = Vec::with_capacity(1024);
|
||||
let mut buffer = [0_u8; 4096];
|
||||
loop {
|
||||
let count = stream
|
||||
.read(&mut buffer)
|
||||
.map_err(|error| format!("read Firecracker API response {path}: {error}"))?;
|
||||
if count == 0 {
|
||||
return if response.is_empty() {
|
||||
Err(format!("empty Firecracker API response for {path}"))
|
||||
} else {
|
||||
Ok(response)
|
||||
};
|
||||
}
|
||||
response.extend_from_slice(&buffer[..count]);
|
||||
if response.len() > MAX_API_RESPONSE_BYTES {
|
||||
return Err(format!(
|
||||
"Firecracker API response for {path} exceeds {MAX_API_RESPONSE_BYTES} bytes"
|
||||
));
|
||||
}
|
||||
let Some(header_end) = response.windows(4).position(|window| window == b"\r\n\r\n") else {
|
||||
continue;
|
||||
};
|
||||
let body_start = header_end + 4;
|
||||
let headers = String::from_utf8_lossy(&response[..header_end]);
|
||||
let content_length = headers.lines().find_map(|line| {
|
||||
let (name, value) = line.split_once(':')?;
|
||||
name.eq_ignore_ascii_case("content-length")
|
||||
.then(|| value.trim().parse::<usize>().ok())
|
||||
.flatten()
|
||||
});
|
||||
match content_length {
|
||||
Some(length) if response.len() >= body_start + length => {
|
||||
response.truncate(body_start + length);
|
||||
return Ok(response);
|
||||
}
|
||||
Some(_) => {}
|
||||
None => return Ok(response),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn check_http_status(path: &str, response: &[u8]) -> Result<(), String> {
|
||||
let text = String::from_utf8_lossy(response);
|
||||
let status_line = text
|
||||
.lines()
|
||||
.next()
|
||||
.ok_or_else(|| format!("empty Firecracker API response for {path}"))?;
|
||||
let status = status_line
|
||||
.split_whitespace()
|
||||
.nth(1)
|
||||
.and_then(|value| value.parse::<u16>().ok())
|
||||
.ok_or_else(|| format!("invalid Firecracker API response for {path}: {status_line}"))?;
|
||||
if (200..300).contains(&status) {
|
||||
Ok(())
|
||||
} else {
|
||||
let body = text.split_once("\r\n\r\n").map_or("", |(_, body)| body);
|
||||
Err(format!(
|
||||
"Firecracker API request {path} failed with status {status}: {}",
|
||||
body.trim()
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_stale_socket(path: &Path) -> Result<(), String> {
|
||||
match std::fs::remove_file(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(format!("remove stale socket {}: {error}", path.display())),
|
||||
}
|
||||
}
|
||||
|
||||
fn terminate_child(child: &mut Child) {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn config() -> FirecrackerLaunchConfig {
|
||||
FirecrackerLaunchConfig {
|
||||
firecracker_binary: PathBuf::from("/firecracker"),
|
||||
kernel_image: PathBuf::from("/vmlinux"),
|
||||
root_disk: PathBuf::from("/root.ext4"),
|
||||
run_dir: PathBuf::from("/run/firecracker"),
|
||||
console_output: PathBuf::from("/run/firecracker/console.log"),
|
||||
guest_init: "/opt/openshell/bin/openshell-driver-firecracker".to_string(),
|
||||
vcpus: 2,
|
||||
mem_mib: 512,
|
||||
vsock_cid: 4,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn configures_no_network_device() {
|
||||
let config = config();
|
||||
let requests = configuration_requests(&config, Path::new("/tmp/vsock.sock"));
|
||||
let paths = requests
|
||||
.iter()
|
||||
.map(|request| request.path)
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(
|
||||
paths,
|
||||
[
|
||||
"/machine-config",
|
||||
"/boot-source",
|
||||
"/drives/rootfs",
|
||||
"/vsock",
|
||||
"/actions"
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
!requests
|
||||
.iter()
|
||||
.any(|request| request.path == "/network-interfaces")
|
||||
);
|
||||
assert_eq!(requests[2].body["is_read_only"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn boot_source_uses_driver_as_init() {
|
||||
let command_line = kernel_command_line("/opt/openshell/bin/openshell-driver-firecracker");
|
||||
assert!(command_line.contains("root=/dev/vda rw"));
|
||||
assert!(command_line.contains("init=/opt/openshell/bin/openshell-driver-firecracker"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_only_successful_api_statuses() {
|
||||
assert!(check_http_status("/actions", b"HTTP/1.1 204 No Content\r\n\r\n").is_ok());
|
||||
assert!(check_http_status("/actions", b"HTTP/1.1 400 Bad\r\n\r\nnope").is_err());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
# Firecracker smoke test
|
||||
|
||||
This runner boots a real Firecracker VM, drives the RFC 0012 lifecycle from the
|
||||
host, and verifies that the guest process leaf runs the admitted command through
|
||||
the existing OpenShell process supervisor.
|
||||
|
||||
It does not use `sudo`, a TAP interface, a guest NIC, nftables, or
|
||||
`CAP_NET_ADMIN`. The current login session must have read/write access to
|
||||
`/dev/kvm`.
|
||||
|
||||
Default fixtures live in `/tmp/openshell-firecracker-e2e-fixtures`. Override
|
||||
them with:
|
||||
|
||||
```shell
|
||||
OPENSHELL_FIRECRACKER_BINARY=/path/to/firecracker \
|
||||
OPENSHELL_FIRECRACKER_KERNEL_IMAGE=/path/to/vmlinux \
|
||||
OPENSHELL_FIRECRACKER_ROOT_DISK=/path/to/root.ext4 \
|
||||
mise run e2e:firecracker
|
||||
```
|
||||
|
||||
The runner clones the root disk into a temporary directory and injects the
|
||||
current `openshell-driver-firecracker` binary plus a one-time authenticated
|
||||
guest configuration. It never modifies the source fixture.
|
||||
Executable
+174
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Boot the host-supervised Firecracker backend and exercise its RFC 0012
|
||||
# lifecycle against the current process supervisor implementation in the guest.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
FIXTURE_DIR="${OPENSHELL_FIRECRACKER_FIXTURE_DIR:-/tmp/openshell-firecracker-e2e-fixtures}"
|
||||
DRIVER_BIN="${OPENSHELL_FIRECRACKER_DRIVER_BIN:-${ROOT}/target/debug/openshell-driver-firecracker}"
|
||||
FIRECRACKER_BIN="${OPENSHELL_FIRECRACKER_BINARY:-${FIXTURE_DIR}/release/release-v1.16.1-aarch64/firecracker-v1.16.1-aarch64}"
|
||||
KERNEL_IMAGE="${OPENSHELL_FIRECRACKER_KERNEL_IMAGE:-${FIXTURE_DIR}/vmlinux-6.1.155}"
|
||||
ROOT_DISK_FIXTURE="${OPENSHELL_FIRECRACKER_ROOT_DISK:-${FIXTURE_DIR}/openshell-ubuntu-24.04.ext4}"
|
||||
BOOT_TIMEOUT_SECONDS="${OPENSHELL_FIRECRACKER_BOOT_TIMEOUT_SECONDS:-60}"
|
||||
KEEP_STATE="${OPENSHELL_FIRECRACKER_KEEP_STATE:-0}"
|
||||
CONTROL_PORT="${OPENSHELL_FIRECRACKER_CONTROL_PORT:-5500}"
|
||||
|
||||
RUN_DIR="$(mktemp -d /tmp/openshell-firecracker-e2e.XXXXXX)"
|
||||
ROOT_DISK="${RUN_DIR}/root.ext4"
|
||||
TOKEN_FILE="${RUN_DIR}/bootstrap.token"
|
||||
GUEST_CONFIG="${RUN_DIR}/firecracker.json"
|
||||
CONSOLE_LOG="${RUN_DIR}/console.log"
|
||||
LAUNCHER_LOG="${RUN_DIR}/launcher.log"
|
||||
SUPERVISOR_LOG="${RUN_DIR}/supervisor.log"
|
||||
VSOCK_SOCKET="${RUN_DIR}/firecracker-vsock.sock"
|
||||
LAUNCHER_PID=""
|
||||
|
||||
BOUNDARY_ID="firecracker-e2e-$$"
|
||||
VSOCK_CID=$(( ($$ % 60000) + 1024 ))
|
||||
SMOKE_MARKER="openshell-firecracker-e2e-ok-${BOUNDARY_ID}"
|
||||
|
||||
fail() {
|
||||
echo "ERROR: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
launcher_is_running() {
|
||||
[ -n "${LAUNCHER_PID}" ] && kill -0 "${LAUNCHER_PID}" 2>/dev/null
|
||||
}
|
||||
|
||||
print_logs() {
|
||||
local log
|
||||
for log in "${LAUNCHER_LOG}" "${SUPERVISOR_LOG}" "${CONSOLE_LOG}"; do
|
||||
if [ -s "${log}" ]; then
|
||||
echo "=== ${log} ===" >&2
|
||||
tail -n 200 "${log}" >&2 || true
|
||||
echo "=== end ${log} ===" >&2
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
local exit_code=$?
|
||||
set +e
|
||||
if launcher_is_running; then
|
||||
kill -TERM "${LAUNCHER_PID}" 2>/dev/null
|
||||
for _ in 1 2 3 4 5 6 7 8 9 10; do
|
||||
launcher_is_running || break
|
||||
sleep 0.2
|
||||
done
|
||||
launcher_is_running && kill -KILL "${LAUNCHER_PID}" 2>/dev/null
|
||||
fi
|
||||
[ -n "${LAUNCHER_PID}" ] && wait "${LAUNCHER_PID}" 2>/dev/null
|
||||
[ "${exit_code}" -eq 0 ] || print_logs
|
||||
if [ "${KEEP_STATE}" = "1" ]; then
|
||||
echo "Firecracker E2E state preserved at ${RUN_DIR}" >&2
|
||||
else
|
||||
case "${RUN_DIR}" in
|
||||
/tmp/openshell-firecracker-e2e.*) rm -rf -- "${RUN_DIR}" ;;
|
||||
*) echo "Refusing to remove unexpected run directory: ${RUN_DIR}" >&2 ;;
|
||||
esac
|
||||
fi
|
||||
exit "${exit_code}"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
wait_for_console_marker() {
|
||||
local marker="$1"
|
||||
local label="$2"
|
||||
local deadline=$(( SECONDS + BOOT_TIMEOUT_SECONDS ))
|
||||
while [ "${SECONDS}" -lt "${deadline}" ]; do
|
||||
grep -Fq -- "${marker}" "${CONSOLE_LOG}" 2>/dev/null && return 0
|
||||
launcher_is_running || fail "Firecracker exited while waiting for ${label}"
|
||||
sleep 0.2
|
||||
done
|
||||
fail "timed out after ${BOOT_TIMEOUT_SECONDS}s waiting for ${label}"
|
||||
}
|
||||
|
||||
inject_file() {
|
||||
local source="$1"
|
||||
local destination="$2"
|
||||
debugfs -w -R "rm ${destination}" "${ROOT_DISK}" >/dev/null 2>&1 || true
|
||||
debugfs -w -R "write ${source} ${destination}" "${ROOT_DISK}" >/dev/null
|
||||
}
|
||||
|
||||
[ "$(uname -s)" = "Linux" ] || fail "Firecracker E2E requires Linux"
|
||||
[ -r /dev/kvm ] && [ -w /dev/kvm ] || \
|
||||
fail "/dev/kvm is not readable and writable; start a login session with the kvm supplementary group"
|
||||
[ -x "${FIRECRACKER_BIN}" ] || fail "Firecracker binary is not executable: ${FIRECRACKER_BIN}"
|
||||
[ -f "${KERNEL_IMAGE}" ] || fail "kernel image not found: ${KERNEL_IMAGE}"
|
||||
[ -f "${ROOT_DISK_FIXTURE}" ] || fail "root disk fixture not found: ${ROOT_DISK_FIXTURE}"
|
||||
[[ "${BOOT_TIMEOUT_SECONDS}" =~ ^[1-9][0-9]*$ ]] || fail "boot timeout must be positive"
|
||||
[[ "${CONTROL_PORT}" =~ ^[1-9][0-9]*$ ]] || fail "control port must be positive"
|
||||
|
||||
for tool in cargo cp debugfs grep jq openssl tail; do
|
||||
command -v "${tool}" >/dev/null 2>&1 || fail "required host tool not found: ${tool}"
|
||||
done
|
||||
|
||||
if [ -n "${RUSTC_WRAPPER:-}" ] && [ "${OPENSHELL_E2E_FIRECRACKER_ALLOW_RUSTC_WRAPPER:-0}" != "1" ]; then
|
||||
unset RUSTC_WRAPPER
|
||||
fi
|
||||
if [ -z "${OPENSHELL_FIRECRACKER_DRIVER_BIN:-}" ]; then
|
||||
echo "==> Building standalone Firecracker driver"
|
||||
cargo build --package openshell-driver-firecracker
|
||||
fi
|
||||
[ -x "${DRIVER_BIN}" ] || fail "driver binary not found: ${DRIVER_BIN}"
|
||||
|
||||
echo "==> Preparing an isolated guest disk"
|
||||
cp --reflink=auto "${ROOT_DISK_FIXTURE}" "${ROOT_DISK}"
|
||||
umask 077
|
||||
openssl rand -hex 32 >"${TOKEN_FILE}"
|
||||
jq -n \
|
||||
--arg boundary_id "${BOUNDARY_ID}" \
|
||||
--arg bootstrap_token "$(tr -d '\n' <"${TOKEN_FILE}")" \
|
||||
--argjson control_port "${CONTROL_PORT}" \
|
||||
'{boundary_id: $boundary_id, bootstrap_token: $bootstrap_token, control_port: $control_port}' \
|
||||
>"${GUEST_CONFIG}"
|
||||
|
||||
debugfs -w -R "mkdir /etc/openshell" "${ROOT_DISK}" >/dev/null 2>&1 || true
|
||||
inject_file "${GUEST_CONFIG}" /etc/openshell/firecracker.json
|
||||
inject_file "${DRIVER_BIN}" /opt/openshell/bin/openshell-driver-firecracker
|
||||
debugfs -w -R \
|
||||
"set_inode_field /opt/openshell/bin/openshell-driver-firecracker mode 0100755" \
|
||||
"${ROOT_DISK}" >/dev/null
|
||||
|
||||
echo "==> Starting Firecracker guest ${BOUNDARY_ID} (vsock only, no NIC)"
|
||||
"${DRIVER_BIN}" launch \
|
||||
--firecracker-binary "${FIRECRACKER_BIN}" \
|
||||
--kernel-image "${KERNEL_IMAGE}" \
|
||||
--root-disk "${ROOT_DISK}" \
|
||||
--run-dir "${RUN_DIR}" \
|
||||
--console-output "${CONSOLE_LOG}" \
|
||||
--vsock-cid "${VSOCK_CID}" \
|
||||
>"${LAUNCHER_LOG}" 2>&1 &
|
||||
LAUNCHER_PID=$!
|
||||
|
||||
wait_for_console_marker \
|
||||
"Firecracker process supervisor leaf listening on vsock port ${CONTROL_PORT}" \
|
||||
"guest process leaf"
|
||||
[ -S "${VSOCK_SOCKET}" ] || fail "Firecracker vsock UDS was not created: ${VSOCK_SOCKET}"
|
||||
|
||||
echo "==> Driving attach, confirm, start_agent, and wait from the host"
|
||||
"${DRIVER_BIN}" supervise \
|
||||
--boundary-id "${BOUNDARY_ID}" \
|
||||
--vsock-uds-path "${VSOCK_SOCKET}" \
|
||||
--vsock-port "${CONTROL_PORT}" \
|
||||
--bootstrap-token-file "${TOKEN_FILE}" \
|
||||
--workdir /sandbox \
|
||||
--timeout-seconds 30 \
|
||||
-- /bin/sh -lc "printf '%s\\n' '${SMOKE_MARKER}'" \
|
||||
>"${SUPERVISOR_LOG}" 2>&1
|
||||
|
||||
grep -Fq 'agent exited: Exited(0)' "${SUPERVISOR_LOG}" || \
|
||||
fail "host supervisor did not observe a successful agent exit"
|
||||
wait_for_console_marker "${SMOKE_MARKER}" "agent output"
|
||||
|
||||
echo "==> Firecracker host-supervisor E2E passed"
|
||||
echo " boundary: ${BOUNDARY_ID}"
|
||||
echo " lifecycle: attach -> confirm -> start_agent -> wait"
|
||||
echo " process leaf: openshell-supervisor-process"
|
||||
echo " transport: authenticated Firecracker virtio-vsock"
|
||||
echo " network: no guest NIC"
|
||||
@@ -155,6 +155,10 @@ run = "e2e/rust/e2e-kubernetes.sh"
|
||||
description = "Start openshell-gateway with the VM compute driver and run VM e2e tests"
|
||||
run = "e2e/rust/e2e-vm.sh"
|
||||
|
||||
["e2e:firecracker"]
|
||||
description = "Boot Firecracker and run the host-supervised RFC 0012 lifecycle smoke test"
|
||||
run = "e2e/firecracker/smoke.sh"
|
||||
|
||||
["e2e:docker"]
|
||||
description = "Run smoke e2e against a standalone gateway with the Docker compute driver"
|
||||
run = "e2e/rust/e2e-docker.sh"
|
||||
|
||||
Reference in New Issue
Block a user