mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-04 16:39:35 +08:00
feat(vm): derive guest rootfs from sandbox images (#957)
This commit is contained in:
@@ -75,69 +75,9 @@ jobs:
|
||||
path: runtime-artifacts/vm-runtime-*.tar.zst
|
||||
retention-days: 1
|
||||
|
||||
build-rootfs:
|
||||
name: Build Rootfs (${{ matrix.arch }})
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
- arch: arm64
|
||||
runner: build-arm64
|
||||
guest_arch: aarch64
|
||||
- arch: amd64
|
||||
runner: build-amd64
|
||||
guest_arch: x86_64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
container:
|
||||
image: ghcr.io/nvidia/openshell/ci:latest
|
||||
credentials:
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
options: --privileged
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
env:
|
||||
MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
OPENSHELL_IMAGE_TAG: ${{ inputs['image-tag'] }}
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ inputs['checkout-ref'] }}
|
||||
|
||||
- name: Mark workspace safe for git
|
||||
run: git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
||||
|
||||
- name: Log in to GHCR
|
||||
run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin
|
||||
|
||||
- name: Install tools
|
||||
run: mise install --locked
|
||||
|
||||
- name: Install zstd
|
||||
run: apt-get update && apt-get install -y --no-install-recommends zstd && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
- name: Build base rootfs tarball
|
||||
run: |
|
||||
set -euo pipefail
|
||||
crates/openshell-vm/scripts/build-rootfs.sh \
|
||||
--base \
|
||||
--arch ${{ matrix.guest_arch }} \
|
||||
target/rootfs-build
|
||||
|
||||
mkdir -p target/vm-runtime-compressed
|
||||
tar -C target/rootfs-build -cf - . \
|
||||
| zstd -19 -T0 -o target/vm-runtime-compressed/rootfs.tar.zst
|
||||
|
||||
- name: Upload rootfs artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: driver-vm-rootfs-${{ matrix.arch }}
|
||||
path: target/vm-runtime-compressed/rootfs.tar.zst
|
||||
retention-days: 1
|
||||
|
||||
build-driver-vm-linux:
|
||||
name: Build Driver VM (Linux ${{ matrix.arch }})
|
||||
needs: [download-kernel-runtime, build-rootfs]
|
||||
needs: [download-kernel-runtime]
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -145,10 +85,12 @@ jobs:
|
||||
runner: build-arm64
|
||||
target: aarch64-unknown-linux-gnu
|
||||
platform: linux-aarch64
|
||||
guest_arch: aarch64
|
||||
- arch: amd64
|
||||
runner: build-amd64
|
||||
target: x86_64-unknown-linux-gnu
|
||||
platform: linux-x86_64
|
||||
guest_arch: x86_64
|
||||
runs-on: ${{ matrix.runner }}
|
||||
timeout-minutes: 30
|
||||
container:
|
||||
@@ -192,12 +134,6 @@ jobs:
|
||||
name: driver-vm-kernel-runtime-tarballs
|
||||
path: runtime-download/
|
||||
|
||||
- name: Download rootfs tarball
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: driver-vm-rootfs-${{ matrix.arch }}
|
||||
path: rootfs-download/
|
||||
|
||||
- name: Stage compressed runtime for embedding
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -208,6 +144,9 @@ jobs:
|
||||
zstd -d "runtime-download/vm-runtime-${{ matrix.platform }}.tar.zst" --stdout \
|
||||
| tar -xf - -C "$EXTRACT_DIR"
|
||||
|
||||
echo "Extracted runtime files:"
|
||||
ls -lah "$EXTRACT_DIR"
|
||||
|
||||
for file in "$EXTRACT_DIR"/*; do
|
||||
[ -f "$file" ] || continue
|
||||
name=$(basename "$file")
|
||||
@@ -215,9 +154,22 @@ jobs:
|
||||
zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file"
|
||||
done
|
||||
|
||||
cp rootfs-download/rootfs.tar.zst "${COMPRESSED_DIR}/rootfs.tar.zst"
|
||||
echo "Staged compressed runtime artifacts:"
|
||||
ls -lah "$COMPRESSED_DIR"
|
||||
|
||||
- name: Build bundled supervisor
|
||||
run: |
|
||||
set -euo pipefail
|
||||
OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" \
|
||||
tasks/scripts/vm/build-supervisor-bundle.sh --arch "${{ matrix.guest_arch }}"
|
||||
|
||||
- name: Verify embedded driver inputs
|
||||
run: |
|
||||
set -euo pipefail
|
||||
for file in libkrun.so.zst libkrunfw.so.5.zst gvproxy.zst openshell-sandbox.zst; do
|
||||
test -s "target/vm-runtime-compressed/${file}"
|
||||
done
|
||||
|
||||
- name: Scope workspace to driver-vm crates
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
@@ -421,7 +421,7 @@ jobs:
|
||||
# ---------------------------------------------------------------------------
|
||||
build-driver-vm-linux:
|
||||
name: Build Driver VM (Linux ${{ matrix.arch }})
|
||||
needs: [compute-versions, download-kernel-runtime, build-rootfs]
|
||||
needs: [compute-versions, download-kernel-runtime]
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -477,12 +477,6 @@ jobs:
|
||||
name: kernel-runtime-tarballs
|
||||
path: runtime-download/
|
||||
|
||||
- name: Download rootfs tarball
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: rootfs-${{ matrix.arch }}
|
||||
path: rootfs-download/
|
||||
|
||||
- name: Stage compressed runtime for embedding
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -504,12 +498,15 @@ jobs:
|
||||
zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file"
|
||||
done
|
||||
|
||||
# Copy rootfs tarball (already zstd-compressed)
|
||||
cp rootfs-download/rootfs.tar.zst "${COMPRESSED_DIR}/rootfs.tar.zst"
|
||||
|
||||
echo "Staged compressed artifacts:"
|
||||
ls -lah "$COMPRESSED_DIR"
|
||||
|
||||
- name: Build bundled supervisor
|
||||
run: |
|
||||
set -euo pipefail
|
||||
OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${PWD}/target/vm-runtime-compressed" \
|
||||
tasks/scripts/vm/build-supervisor-bundle.sh --arch "${{ matrix.guest_arch }}"
|
||||
|
||||
- name: Scope workspace to driver-vm crates
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -551,7 +548,7 @@ jobs:
|
||||
# ---------------------------------------------------------------------------
|
||||
build-driver-vm-macos:
|
||||
name: Build Driver VM (macOS)
|
||||
needs: [compute-versions, download-kernel-runtime, build-rootfs]
|
||||
needs: [compute-versions, download-kernel-runtime]
|
||||
runs-on: build-amd64
|
||||
timeout-minutes: 60
|
||||
container:
|
||||
@@ -591,12 +588,6 @@ jobs:
|
||||
name: kernel-runtime-tarballs
|
||||
path: runtime-download/
|
||||
|
||||
- name: Download rootfs tarball (arm64)
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: rootfs-arm64
|
||||
path: rootfs-download/
|
||||
|
||||
- name: Prepare compressed runtime directory
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -619,12 +610,24 @@ jobs:
|
||||
zstd -19 -f -q -T0 -o "${COMPRESSED_DIR}/${name}.zst" "$file"
|
||||
done
|
||||
|
||||
# The macOS VM guest is always Linux ARM64, so use the arm64 rootfs
|
||||
cp rootfs-download/rootfs.tar.zst "${COMPRESSED_DIR}/rootfs.tar.zst"
|
||||
|
||||
echo "Staged macOS compressed artifacts:"
|
||||
ls -lah "$COMPRESSED_DIR"
|
||||
|
||||
- name: Build bundled supervisor
|
||||
run: |
|
||||
set -euo pipefail
|
||||
docker buildx build \
|
||||
--file deploy/docker/Dockerfile.images \
|
||||
--platform linux/arm64 \
|
||||
--build-arg OPENSHELL_CARGO_VERSION="${{ needs.compute-versions.outputs.cargo_version }}" \
|
||||
--build-arg OPENSHELL_IMAGE_TAG=dev \
|
||||
--target supervisor-output \
|
||||
--output type=local,dest=supervisor-out/ \
|
||||
.
|
||||
|
||||
zstd -19 -T0 -f supervisor-out/openshell-sandbox \
|
||||
-o "${PWD}/target/vm-runtime-compressed-macos/openshell-sandbox.zst"
|
||||
|
||||
- name: Build macOS binary via Docker (osxcross)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
@@ -776,7 +779,7 @@ jobs:
|
||||
|
||||
### VM Compute Driver Binaries
|
||||
|
||||
`openshell-driver-vm` binaries with embedded kernel runtime and sandbox rootfs.
|
||||
`openshell-driver-vm` binaries with embedded kernel runtime and bundled sandbox supervisor.
|
||||
Launched by the gateway when `--drivers=vm` is configured. Rebuilt on every
|
||||
push to main alongside the openshell-vm binaries.
|
||||
|
||||
|
||||
Generated
+439
-8
@@ -621,6 +621,12 @@ dependencies = [
|
||||
"shlex",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cesu8"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c"
|
||||
|
||||
[[package]]
|
||||
name = "cexpr"
|
||||
version = "0.6.0"
|
||||
@@ -761,6 +767,16 @@ version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
|
||||
|
||||
[[package]]
|
||||
name = "combine"
|
||||
version = "4.6.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "compact_str"
|
||||
version = "0.7.1"
|
||||
@@ -808,6 +824,27 @@ version = "0.10.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c"
|
||||
|
||||
[[package]]
|
||||
name = "const_format"
|
||||
version = "0.2.36"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4481a617ad9a412be3b97c5d403fef8ed023103368908b9c50af598ff467cc1e"
|
||||
dependencies = [
|
||||
"const_format_proc_macros",
|
||||
"konst",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const_format_proc_macros"
|
||||
version = "0.2.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d57c2eccfb16dbac1f4e61e206105db5820c9d26c3c472bc17c774259ef7744"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-xid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "constant_time_eq"
|
||||
version = "0.4.2"
|
||||
@@ -1175,6 +1212,37 @@ dependencies = [
|
||||
"syn 1.0.109",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder"
|
||||
version = "0.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
|
||||
dependencies = [
|
||||
"derive_builder_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_core"
|
||||
version = "0.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "derive_builder_macro"
|
||||
version = "0.20.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
|
||||
dependencies = [
|
||||
"derive_builder_core",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dialoguer"
|
||||
version = "0.11.0"
|
||||
@@ -1648,6 +1716,18 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getset"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9cf0fc11e47561d47397154977bc219f4cf809b2974facc3ccb3b89e2436f912"
|
||||
dependencies = [
|
||||
"proc-macro-error2",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ghash"
|
||||
version = "0.5.1"
|
||||
@@ -1861,6 +1941,15 @@ dependencies = [
|
||||
"itoa",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-auth"
|
||||
version = "0.1.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "150fa4a9462ef926824cf4519c84ed652ca8f4fbae34cb8af045b5cbcaf98822"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "http-body"
|
||||
version = "1.0.1"
|
||||
@@ -2324,6 +2413,50 @@ version = "1.0.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682"
|
||||
|
||||
[[package]]
|
||||
name = "jni"
|
||||
version = "0.21.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97"
|
||||
dependencies = [
|
||||
"cesu8",
|
||||
"cfg-if",
|
||||
"combine",
|
||||
"jni-sys 0.3.1",
|
||||
"log",
|
||||
"thiserror 1.0.69",
|
||||
"walkdir",
|
||||
"windows-sys 0.45.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-sys"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258"
|
||||
dependencies = [
|
||||
"jni-sys 0.4.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-sys"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
|
||||
dependencies = [
|
||||
"jni-sys-macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jni-sys-macros"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.34"
|
||||
@@ -2387,6 +2520,20 @@ dependencies = [
|
||||
"simple_asn1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jsonwebtoken"
|
||||
version = "10.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"getrandom 0.2.17",
|
||||
"js-sys",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"signature 2.2.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "k8s-openapi"
|
||||
version = "0.21.1"
|
||||
@@ -2400,6 +2547,21 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "konst"
|
||||
version = "0.2.20"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "128133ed7824fcd73d6e7b17957c5eb7bacb885649bd8c69708b2331a10bcefb"
|
||||
dependencies = [
|
||||
"konst_macro_rules",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "konst_macro_rules"
|
||||
version = "0.2.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a4933f3f57a8e9d9da04db23fb153356ecaf00cbd14aee46279c33dc80925c37"
|
||||
|
||||
[[package]]
|
||||
name = "kube"
|
||||
version = "0.90.0"
|
||||
@@ -3052,7 +3214,7 @@ dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
"http",
|
||||
"rand 0.8.6",
|
||||
"reqwest",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_path_to_error",
|
||||
@@ -3070,6 +3232,60 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oci-client"
|
||||
version = "0.16.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1b7f8deaffcd3b0e3baf93dddcab3d18b91d46dc37d38a8b170089b234de5bb3"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-auth",
|
||||
"jsonwebtoken 10.3.0",
|
||||
"lazy_static",
|
||||
"oci-spec",
|
||||
"olpc-cjson",
|
||||
"regex",
|
||||
"reqwest 0.13.2",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.18",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"unicase",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "oci-spec"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8445a2631507cec628a15fdd6154b54a3ab3f20ed4fe9d73a3b8b7a4e1ba03a"
|
||||
dependencies = [
|
||||
"const_format",
|
||||
"derive_builder",
|
||||
"getset",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum 0.27.2",
|
||||
"strum_macros 0.27.2",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "olpc-cjson"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "696183c9b5fe81a7715d074fd632e8bd46f4ccc0231a3ed7fc580a80de5f7083"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"unicode-normalization",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.4"
|
||||
@@ -3137,7 +3353,7 @@ dependencies = [
|
||||
"owo-colors",
|
||||
"prost-types",
|
||||
"rcgen",
|
||||
"reqwest",
|
||||
"reqwest 0.12.28",
|
||||
"rustls",
|
||||
"rustls-pemfile",
|
||||
"serde",
|
||||
@@ -3240,18 +3456,22 @@ dependencies = [
|
||||
name = "openshell-driver-vm"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"bollard",
|
||||
"clap",
|
||||
"flate2",
|
||||
"futures",
|
||||
"libc",
|
||||
"libloading",
|
||||
"miette",
|
||||
"nix",
|
||||
"oci-client",
|
||||
"openshell-core",
|
||||
"openshell-vfio",
|
||||
"polling",
|
||||
"prost-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"tar",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
@@ -3311,7 +3531,7 @@ version = "0.0.0"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"openshell-core",
|
||||
"reqwest",
|
||||
"reqwest 0.12.28",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yml",
|
||||
@@ -3387,7 +3607,7 @@ dependencies = [
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
"ipnet",
|
||||
"jsonwebtoken",
|
||||
"jsonwebtoken 9.3.1",
|
||||
"metrics",
|
||||
"metrics-exporter-prometheus",
|
||||
"miette",
|
||||
@@ -3404,7 +3624,7 @@ dependencies = [
|
||||
"prost-types",
|
||||
"rand 0.9.4",
|
||||
"rcgen",
|
||||
"reqwest",
|
||||
"reqwest 0.12.28",
|
||||
"russh",
|
||||
"rustls",
|
||||
"rustls-pemfile",
|
||||
@@ -4062,6 +4282,7 @@ version = "0.11.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"bytes",
|
||||
"getrandom 0.3.4",
|
||||
"lru-slab",
|
||||
@@ -4200,7 +4421,7 @@ dependencies = [
|
||||
"lru",
|
||||
"paste",
|
||||
"stability",
|
||||
"strum",
|
||||
"strum 0.26.3",
|
||||
"unicode-segmentation",
|
||||
"unicode-truncate",
|
||||
"unicode-width 0.1.14",
|
||||
@@ -4335,6 +4556,47 @@ dependencies = [
|
||||
"webpki-roots 1.0.7",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.13.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab3f43e3283ab1488b624b44b0e988d0acea0b3214e694730a055cb6b2efa801"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"hyper-rustls",
|
||||
"hyper-util",
|
||||
"js-sys",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"quinn",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"rustls-platform-verifier",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tokio-util",
|
||||
"tower 0.5.3",
|
||||
"tower-http 0.6.8",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"wasm-streams",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rfc6979"
|
||||
version = "0.4.0"
|
||||
@@ -4538,6 +4800,7 @@ version = "0.23.38"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
"once_cell",
|
||||
"ring",
|
||||
@@ -4578,12 +4841,40 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-platform-verifier"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784"
|
||||
dependencies = [
|
||||
"core-foundation",
|
||||
"core-foundation-sys",
|
||||
"jni",
|
||||
"log",
|
||||
"once_cell",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"rustls-platform-verifier-android",
|
||||
"rustls-webpki",
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustls-platform-verifier-android"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted 0.9.0",
|
||||
@@ -4610,6 +4901,15 @@ dependencies = [
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "same-file"
|
||||
version = "1.0.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
|
||||
dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.29"
|
||||
@@ -5333,9 +5633,15 @@ version = "0.26.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
|
||||
dependencies = [
|
||||
"strum_macros",
|
||||
"strum_macros 0.26.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf"
|
||||
|
||||
[[package]]
|
||||
name = "strum_macros"
|
||||
version = "0.26.4"
|
||||
@@ -5349,6 +5655,18 @@ dependencies = [
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strum_macros"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.117",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
@@ -6001,6 +6319,12 @@ version = "0.1.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
|
||||
|
||||
[[package]]
|
||||
name = "unicase"
|
||||
version = "2.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-bidi"
|
||||
version = "0.3.18"
|
||||
@@ -6157,6 +6481,16 @@ version = "0.9.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
|
||||
|
||||
[[package]]
|
||||
name = "walkdir"
|
||||
version = "2.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
|
||||
dependencies = [
|
||||
"same-file",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "want"
|
||||
version = "0.3.1"
|
||||
@@ -6273,6 +6607,19 @@ dependencies = [
|
||||
"wasmparser",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-streams"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasmparser"
|
||||
version = "0.244.0"
|
||||
@@ -6305,6 +6652,15 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "0.26.11"
|
||||
@@ -6349,6 +6705,15 @@ version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
@@ -6456,6 +6821,15 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.45.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0"
|
||||
dependencies = [
|
||||
"windows-targets 0.42.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-sys"
|
||||
version = "0.48.0"
|
||||
@@ -6501,6 +6875,21 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071"
|
||||
dependencies = [
|
||||
"windows_aarch64_gnullvm 0.42.2",
|
||||
"windows_aarch64_msvc 0.42.2",
|
||||
"windows_i686_gnu 0.42.2",
|
||||
"windows_i686_msvc 0.42.2",
|
||||
"windows_x86_64_gnu 0.42.2",
|
||||
"windows_x86_64_gnullvm 0.42.2",
|
||||
"windows_x86_64_msvc 0.42.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-targets"
|
||||
version = "0.48.5"
|
||||
@@ -6558,6 +6947,12 @@ dependencies = [
|
||||
"windows-link",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_gnullvm"
|
||||
version = "0.48.5"
|
||||
@@ -6576,6 +6971,12 @@ version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
|
||||
|
||||
[[package]]
|
||||
name = "windows_aarch64_msvc"
|
||||
version = "0.48.5"
|
||||
@@ -6594,6 +6995,12 @@ version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_gnu"
|
||||
version = "0.48.5"
|
||||
@@ -6624,6 +7031,12 @@ version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
|
||||
|
||||
[[package]]
|
||||
name = "windows_i686_msvc"
|
||||
version = "0.48.5"
|
||||
@@ -6642,6 +7055,12 @@ version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnu"
|
||||
version = "0.48.5"
|
||||
@@ -6660,6 +7079,12 @@ version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_gnullvm"
|
||||
version = "0.48.5"
|
||||
@@ -6678,6 +7103,12 @@ version = "0.53.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.42.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"
|
||||
|
||||
[[package]]
|
||||
name = "windows_x86_64_msvc"
|
||||
version = "0.48.5"
|
||||
@@ -6887,7 +7318,7 @@ dependencies = [
|
||||
"bindgen",
|
||||
"cmake",
|
||||
"pkg-config",
|
||||
"reqwest",
|
||||
"reqwest 0.12.28",
|
||||
"serde_json",
|
||||
"zip",
|
||||
]
|
||||
|
||||
@@ -20,8 +20,9 @@ kernel.
|
||||
The driver is spawned by `openshell-gateway` as a subprocess, talks to it over a
|
||||
Unix domain socket (`compute-driver.sock`) with the
|
||||
`openshell.compute.v1.ComputeDriver` gRPC surface, and manages per-sandbox
|
||||
microVMs. The runtime (libkrun + libkrunfw + gvproxy) and the sandbox rootfs are
|
||||
embedded directly in the driver binary — no sibling files required at runtime.
|
||||
microVMs. The runtime (libkrun + libkrunfw + gvproxy) and the sandbox
|
||||
supervisor are embedded directly in the driver binary; each sandbox guest
|
||||
rootfs is derived from a container image at create time.
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -30,7 +31,7 @@ graph TD
|
||||
subgraph Host["Host (macOS / Linux)"]
|
||||
GATEWAY["openshell-gateway<br/>(compute::vm::spawn)"]
|
||||
DRIVER["openshell-driver-vm<br/>(compute-driver.sock)"]
|
||||
EMB["Embedded runtime (zstd)<br/>libkrun · libkrunfw · gvproxy<br/>+ sandbox rootfs.tar.zst"]
|
||||
EMB["Embedded runtime (zstd)<br/>libkrun · libkrunfw · gvproxy<br/>+ openshell-sandbox.zst"]
|
||||
GVP["gvproxy (per sandbox)<br/>virtio-net · DHCP · DNS"]
|
||||
|
||||
GATEWAY <-->|gRPC over UDS| DRIVER
|
||||
@@ -58,8 +59,8 @@ never binds a host-side TCP listener.
|
||||
|
||||
## Embedded Runtime
|
||||
|
||||
`openshell-driver-vm` embeds the VM runtime libraries and the sandbox rootfs as
|
||||
zstd-compressed byte arrays, extracting on demand:
|
||||
`openshell-driver-vm` embeds the VM runtime libraries and the sandbox
|
||||
supervisor as zstd-compressed byte arrays, extracting on demand:
|
||||
|
||||
```text
|
||||
~/.local/share/openshell/vm-runtime/<version>/ # libkrun / libkrunfw / gvproxy
|
||||
@@ -74,14 +75,20 @@ Old runtime cache versions are cleaned up when a new version is extracted.
|
||||
|
||||
### Sandbox rootfs preparation
|
||||
|
||||
The rootfs tarball the driver embeds starts from the same minimal Ubuntu base
|
||||
used across the project, and is **rewritten into a supervisor-only sandbox
|
||||
guest** during extraction:
|
||||
Each VM sandbox starts from either a registry image fetched directly over OCI or
|
||||
a local Docker image reference produced by Dockerfile-based `--from` sources.
|
||||
For local Dockerfile sources, the CLI builds the image on the local Docker
|
||||
daemon and passes the ordinary image tag through `template.image`. The VM driver
|
||||
first checks the local Docker daemon for that tag; when present, it exports the
|
||||
image filesystem and **rewrites that filesystem into a supervisor-only sandbox
|
||||
guest** before caching it:
|
||||
|
||||
- k3s state and Kubernetes manifests are stripped out
|
||||
- `/srv/openshell-vm-sandbox-init.sh` is installed as the guest entrypoint
|
||||
- the guest boots directly into `openshell-sandbox` — no k3s, no kube-proxy,
|
||||
no CNI plugins
|
||||
- the bundled `openshell-sandbox` binary is copied into
|
||||
`/opt/openshell/bin/openshell-sandbox`
|
||||
- k3s state and Kubernetes manifests are stripped out if the image contains them
|
||||
- the guest boots directly into `openshell-sandbox` — no k3s, no kube-proxy, no
|
||||
CNI plugins
|
||||
|
||||
See `crates/openshell-driver-vm/src/rootfs.rs` for the rewrite logic and
|
||||
`crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh` for the init
|
||||
@@ -95,6 +102,48 @@ spawns one launcher per sandbox as a subprocess, which in turn starts `gvproxy`
|
||||
and calls `krun_start_enter` to boot the guest. Keeping the launcher in the
|
||||
same binary means the driver ships a single artifact for both roles.
|
||||
|
||||
When a sandbox sets `template.image` through `openshell sandbox create --from ...`,
|
||||
the VM driver treats that image as the base guest rootfs source for that
|
||||
sandbox. When `template.image` is omitted, the gateway fills it from the VM
|
||||
driver's advertised `default_image`, which matches the gateway's configured
|
||||
sandbox image. The driver:
|
||||
|
||||
- resolves the image on the gateway host without Docker for registry and
|
||||
community image refs
|
||||
- for local Dockerfile sources, the CLI builds through the host Docker socket
|
||||
and passes the resulting ordinary Docker tag through `template.image`
|
||||
- unpacks the image filesystem, injects the VM sandbox init/supervisor files,
|
||||
and validates required guest tools such as `bash`, `mount`, `ip`, and `sed`
|
||||
- caches the prepared guest rootfs under
|
||||
`<vm-driver-state-dir>/images/<image-identity>/rootfs.tar`
|
||||
- extracts a private runtime copy under
|
||||
`<vm-driver-state-dir>/sandboxes/<sandbox-id>/rootfs`
|
||||
|
||||
The cache key uses an immutable image identity: repo digest for registry images
|
||||
and the local Docker image ID for images resolved from the local daemon.
|
||||
Different VM sandboxes can use different base images concurrently because the
|
||||
shared cache is per image, not global for the driver. Cached prepared rootfs
|
||||
entries remain on disk until the operator removes them from the VM driver state
|
||||
directory.
|
||||
|
||||
Docker is therefore no longer required for VM sandboxes created from registry or
|
||||
community image refs. It is only required on the local CLI/gateway host when the
|
||||
source is a local Dockerfile or build context.
|
||||
|
||||
Local Dockerfile sources are treated as trusted local-development inputs for VM
|
||||
gateways. Remote VM gateways still reject local Dockerfile sources until a
|
||||
gateway-side artifact validation and transfer boundary is designed.
|
||||
|
||||
There is no embedded guest rootfs fallback anymore. VM sandboxes therefore
|
||||
require either `template.image` or a configured default sandbox image. This is
|
||||
still replace-the-rootfs semantics, so VM images must remain base-compatible
|
||||
with the sandbox guest init path. Distroless or `scratch` images are not
|
||||
expected to work.
|
||||
|
||||
The separate `openshell-vm` binary still uses `vm:rootfs` to build a standalone
|
||||
embedded guest filesystem, but `openshell-driver-vm` no longer consumes that
|
||||
artifact.
|
||||
|
||||
## Network Plane
|
||||
|
||||
The driver launches a **dedicated `gvproxy` instance per sandbox** to provide the
|
||||
@@ -178,8 +227,8 @@ graph LR
|
||||
The `vm-runtime-<platform>.tar.zst` artifact is consumed by
|
||||
`openshell-driver-vm`'s `build.rs`, which embeds the library set into the
|
||||
binary via `include_bytes!()`. Setting `OPENSHELL_VM_RUNTIME_COMPRESSED_DIR`
|
||||
at build time (wired up by `crates/openshell-driver-vm/start.sh`) points the
|
||||
build at the staged artifacts.
|
||||
at build time (wired up by `tasks/scripts/gateway-vm.sh`, registered as
|
||||
`mise run gateway:vm`) points the build at the staged artifacts.
|
||||
|
||||
## Kernel Config Fragment
|
||||
|
||||
@@ -262,8 +311,8 @@ host platform.
|
||||
### Driver Binary (`release-vm-dev.yml`)
|
||||
|
||||
Builds the self-contained `openshell-driver-vm` binary for every platform,
|
||||
with the kernel runtime + sandbox rootfs embedded. Runs on every push to
|
||||
`main` that touches VM-related crates.
|
||||
with the kernel runtime + bundled sandbox supervisor embedded. Runs on every
|
||||
push to `main` that touches VM-related crates.
|
||||
|
||||
The `download-kernel-runtime` job pulls the current `vm-runtime-<platform>.tar.zst`
|
||||
from the `vm-dev` release; the `build-openshell-driver-vm` jobs set
|
||||
@@ -273,14 +322,15 @@ cross-compiled via osxcross (no macOS runner needed for the binary build —
|
||||
only for the kernel build).
|
||||
|
||||
macOS driver binaries produced via osxcross are not codesigned. Development
|
||||
builds are signed automatically by `crates/openshell-driver-vm/start.sh`; a
|
||||
packaged release needs signing in CI.
|
||||
builds are signed automatically by `tasks/scripts/gateway-vm.sh`
|
||||
(registered as `mise run gateway:vm`); a packaged release needs signing in
|
||||
CI.
|
||||
|
||||
## Rollout Strategy
|
||||
|
||||
1. Custom runtime is embedded by default when building `openshell-driver-vm`
|
||||
with `OPENSHELL_VM_RUNTIME_COMPRESSED_DIR` set (wired up by
|
||||
`crates/openshell-driver-vm/start.sh`).
|
||||
`tasks/scripts/gateway-vm.sh`).
|
||||
2. The sandbox init script validates kernel capabilities at boot and fails
|
||||
fast if missing.
|
||||
3. For development, override with `OPENSHELL_VM_RUNTIME_DIR` to use a local
|
||||
|
||||
@@ -139,8 +139,8 @@ All configuration is via CLI flags with environment variable fallbacks. The `--d
|
||||
| `--grpc-endpoint` | `OPENSHELL_GRPC_ENDPOINT` | None | gRPC endpoint reachable from within the cluster (for supervisor callbacks) |
|
||||
| `--drivers` | `OPENSHELL_DRIVERS` | `kubernetes` | Compute backend to use. Current options are `kubernetes`, `docker`, and `vm`. |
|
||||
| `--docker-network-name` | `OPENSHELL_DOCKER_NETWORK_NAME` | `openshell-docker` | Docker bridge network that local Docker sandboxes join |
|
||||
| `--vm-driver-state-dir` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Host directory for VM sandbox rootfs, console logs, and runtime state |
|
||||
| `--driver-dir` | `OPENSHELL_DRIVER_DIR` | unset | Override directory for `openshell-driver-vm`. When unset, the gateway searches `~/.local/libexec/openshell`, `/usr/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`, then a sibling binary. |
|
||||
| `--vm-driver-state-dir` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Host directory for VM sandbox rootfs, console logs, runtime state, and shared image-rootfs cache |
|
||||
| `--vm-krun-log-level` | `OPENSHELL_VM_KRUN_LOG_LEVEL` | `1` | libkrun log level for VM helper processes |
|
||||
| `--vm-driver-vcpus` | `OPENSHELL_VM_DRIVER_VCPUS` | `2` | Default vCPU count for VM sandboxes |
|
||||
| `--vm-driver-mem-mib` | `OPENSHELL_VM_DRIVER_MEM_MIB` | `2048` | Default memory allocation for VM sandboxes in MiB |
|
||||
@@ -625,7 +625,7 @@ The Docker driver (`crates/openshell-driver-docker/src/lib.rs`) is an in-process
|
||||
|
||||
`VmDriver` (`crates/openshell-driver-vm/src/driver.rs`) is served by the standalone `openshell-driver-vm` process. The gateway spawns that binary on demand and talks to it over the internal `openshell.compute.v1.ComputeDriver` gRPC contract via a Unix domain socket.
|
||||
|
||||
- **Create**: The VM driver process allocates a sandbox-specific rootfs from its own embedded `rootfs.tar.zst`, injects an explicitly configured guest mTLS bundle when the gateway callback endpoint is `https://`, then re-execs itself in a hidden helper mode that loads libkrun directly and boots the supervisor.
|
||||
- **Create**: The VM driver process exports the selected sandbox image from the local Docker daemon, rewrites it into a sandbox-specific guest rootfs, injects an explicitly configured guest mTLS bundle when the gateway callback endpoint is `https://`, then re-execs itself in a hidden helper mode that loads libkrun directly and boots the supervisor.
|
||||
- **Networking**: The helper starts an embedded `gvproxy`, wires it into libkrun as virtio-net, and gives the guest outbound connectivity. No inbound TCP listener is needed — the supervisor reaches the gateway over its outbound `ConnectSupervisor` stream.
|
||||
- **Gateway callback**: The guest init script configures `eth0` for gvproxy networking, seeds `/etc/hosts` so `host.openshell.internal` resolves to the gvproxy gateway IP (`192.168.127.1`), preserves gvproxy's legacy `host.containers.internal` / `host.docker.internal` DNS answers, prefers the configured `OPENSHELL_GRPC_ENDPOINT`, and falls back to those aliases or the raw gateway IP when local hostname resolution is unavailable on macOS.
|
||||
- **Guest boot**: The sandbox guest runs a minimal init script that starts `openshell-sandbox` directly as PID 1 inside the VM.
|
||||
|
||||
@@ -19,8 +19,9 @@ The CLI classifies the value in this order:
|
||||
|
||||
1. **Existing file** whose name contains "Dockerfile" (case-insensitive) — treated as a Dockerfile to build.
|
||||
2. **Existing directory** containing a `Dockerfile` — treated as a build context directory.
|
||||
3. **Contains `/`, `:`, or `.`** — treated as a full container image reference.
|
||||
4. **Otherwise** — treated as a community sandbox name, expanded to `{OPENSHELL_COMMUNITY_REGISTRY}/{name}:latest`.
|
||||
3. **Missing explicit local path** (for example `./Dockerfile`, `../ctx`, or an absolute path) — rejected locally instead of sent to the gateway as an image pull.
|
||||
4. **Contains `/`, `:`, or `.`** — treated as a full container image reference.
|
||||
5. **Otherwise** — treated as a community sandbox name, expanded to `{OPENSHELL_COMMUNITY_REGISTRY}/{name}:latest`.
|
||||
|
||||
The community registry prefix defaults to `ghcr.io/nvidia/openshell-community/sandboxes` and can be overridden with the `OPENSHELL_COMMUNITY_REGISTRY` environment variable.
|
||||
|
||||
|
||||
@@ -1,10 +1,13 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Build container images from Dockerfiles.
|
||||
//! Build container images for gateway runtimes.
|
||||
//!
|
||||
//! This module wraps bollard's `build_image()` API to build a container image
|
||||
//! from a Dockerfile and build context into the local Docker daemon.
|
||||
//! from a Dockerfile and build context. Kubernetes deployments reuse the
|
||||
//! existing push pipeline to import the image into the gateway's containerd
|
||||
//! runtime. VM deployments keep the built image in the local Docker daemon and
|
||||
//! pass an internal local-image reference to the VM driver.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
@@ -14,7 +17,15 @@ use bollard::query_parameters::BuildImageOptionsBuilder;
|
||||
use futures::StreamExt;
|
||||
use miette::{IntoDiagnostic, Result, WrapErr};
|
||||
|
||||
/// Build a container image from a Dockerfile into the local Docker daemon.
|
||||
use crate::constants::container_name;
|
||||
use crate::push::push_local_images;
|
||||
|
||||
/// Build a container image from a Dockerfile using the local Docker daemon.
|
||||
///
|
||||
/// This is used by `openshell sandbox create --from <Dockerfile>` for both the
|
||||
/// Kubernetes and VM backends. The image remains available in the local Docker
|
||||
/// daemon so the caller can either hand the resulting tag directly to the VM
|
||||
/// backend or import it into a local gateway containerd runtime.
|
||||
#[allow(clippy::implicit_hasher)]
|
||||
pub async fn build_local_image(
|
||||
dockerfile_path: &Path,
|
||||
@@ -32,6 +43,49 @@ pub async fn build_local_image(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Push a locally-built image into the gateway's containerd runtime.
|
||||
#[allow(clippy::implicit_hasher)]
|
||||
pub async fn push_image_into_gateway(
|
||||
tag: &str,
|
||||
gateway_name: &str,
|
||||
on_log: &mut impl FnMut(String),
|
||||
) -> Result<()> {
|
||||
on_log(format!(
|
||||
"Pushing image {tag} into gateway \"{gateway_name}\""
|
||||
));
|
||||
let local_docker = crate::docker::connect_local_for_large_transfers()
|
||||
.into_diagnostic()
|
||||
.wrap_err("failed to connect to local Docker daemon")?;
|
||||
let container = container_name(gateway_name);
|
||||
let images: Vec<&str> = vec![tag];
|
||||
push_local_images(&local_docker, &local_docker, &container, &images, on_log).await?;
|
||||
|
||||
on_log(format!("Image {tag} is available in the gateway."));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build a container image from a Dockerfile and push it into the gateway.
|
||||
///
|
||||
/// This is used by `openshell sandbox create --from <Dockerfile>` when the
|
||||
/// active gateway is the local Kubernetes deployment. It:
|
||||
/// 1. Creates a tar archive of the build context directory.
|
||||
/// 2. Sends it to the local Docker daemon via `build_image()`.
|
||||
/// 3. Pushes the resulting image into the gateway's containerd via the
|
||||
/// existing `push_local_images()` pipeline.
|
||||
#[allow(clippy::implicit_hasher)]
|
||||
pub async fn build_and_push_image(
|
||||
dockerfile_path: &Path,
|
||||
tag: &str,
|
||||
context_dir: &Path,
|
||||
gateway_name: &str,
|
||||
build_args: &HashMap<String, String>,
|
||||
on_log: &mut impl FnMut(String),
|
||||
) -> Result<()> {
|
||||
build_local_image(dockerfile_path, tag, context_dir, build_args, on_log).await?;
|
||||
push_image_into_gateway(tag, gateway_name, on_log).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Build a container image using the local Docker daemon.
|
||||
///
|
||||
/// Creates a tar archive of `context_dir`, sends it to Docker with the
|
||||
|
||||
@@ -65,6 +65,10 @@ pub struct GatewayMetadata {
|
||||
/// When set, tokens will include these scopes for fine-grained access control.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub oidc_scopes: Option<String>,
|
||||
|
||||
/// Local VM driver state directory for standalone VM gateways.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub vm_driver_state_dir: Option<PathBuf>,
|
||||
}
|
||||
|
||||
impl GatewayMetadata {
|
||||
|
||||
+200
-20
@@ -2758,6 +2758,7 @@ pub async fn sandbox_create(
|
||||
}
|
||||
|
||||
/// Resolved source for the `--from` flag on `sandbox create`.
|
||||
#[derive(Debug)]
|
||||
enum ResolvedSource {
|
||||
/// A ready-to-use container image reference.
|
||||
Image(String),
|
||||
@@ -2774,19 +2775,15 @@ enum ResolvedSource {
|
||||
/// Resolution order:
|
||||
/// 1. Existing file whose name contains "Dockerfile" → build from file.
|
||||
/// 2. Existing directory that contains a `Dockerfile` → build from directory.
|
||||
/// 3. Value contains `/`, `:`, or `.` → treat as a full image reference.
|
||||
/// 4. Otherwise → community sandbox name, expanded via the registry prefix.
|
||||
/// 3. Missing explicit local paths → local error, not image pull.
|
||||
/// 4. Value contains `/`, `:`, or `.` → treat as a full image reference.
|
||||
/// 5. Otherwise → community sandbox name, expanded via the registry prefix.
|
||||
fn resolve_from(value: &str) -> Result<ResolvedSource> {
|
||||
let path = Path::new(value);
|
||||
|
||||
// 1. Existing file that looks like a Dockerfile.
|
||||
if path.is_file() {
|
||||
let name = path
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy())
|
||||
.unwrap_or_default();
|
||||
let lower = name.to_lowercase();
|
||||
if lower.contains("dockerfile") || lower.ends_with(".dockerfile") {
|
||||
if filename_looks_like_dockerfile(path) {
|
||||
let dockerfile = path
|
||||
.canonicalize()
|
||||
.into_diagnostic()
|
||||
@@ -2800,6 +2797,13 @@ fn resolve_from(value: &str) -> Result<ResolvedSource> {
|
||||
context,
|
||||
});
|
||||
}
|
||||
|
||||
if value_looks_like_local_source(value) {
|
||||
return Err(miette::miette!(
|
||||
"local --from file is not a Dockerfile: {}",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Existing directory containing a Dockerfile.
|
||||
@@ -2822,13 +2826,57 @@ fn resolve_from(value: &str) -> Result<ResolvedSource> {
|
||||
));
|
||||
}
|
||||
|
||||
// 3. Full image reference or community sandbox name — delegate to shared
|
||||
if path.exists() {
|
||||
return Err(miette::miette!(
|
||||
"local --from path is not a regular file or directory: {}",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
|
||||
// 3. Missing explicit local paths should fail locally. Otherwise values
|
||||
// like `./Dockerfile` reach the gateway as image references and fail as
|
||||
// Docker pull errors.
|
||||
if value_looks_like_local_source(value) {
|
||||
return Err(miette::miette!(
|
||||
"local --from path does not exist: {}\n\
|
||||
Use an existing Dockerfile, a directory containing Dockerfile, or a container image reference.",
|
||||
path.display()
|
||||
));
|
||||
}
|
||||
|
||||
// 4. Full image reference or community sandbox name — delegate to shared
|
||||
// resolution in openshell-core.
|
||||
Ok(ResolvedSource::Image(
|
||||
openshell_core::image::resolve_community_image(value),
|
||||
))
|
||||
}
|
||||
|
||||
fn filename_looks_like_dockerfile(path: &Path) -> bool {
|
||||
let name = path
|
||||
.file_name()
|
||||
.map(|n| n.to_string_lossy())
|
||||
.unwrap_or_default();
|
||||
let lower = name.to_lowercase();
|
||||
lower.contains("dockerfile") || lower.ends_with(".dockerfile")
|
||||
}
|
||||
|
||||
fn value_looks_like_local_source(value: &str) -> bool {
|
||||
value_is_explicit_local_path(value) || value_looks_like_bare_dockerfile_name(value)
|
||||
}
|
||||
|
||||
fn value_is_explicit_local_path(value: &str) -> bool {
|
||||
let path = Path::new(value);
|
||||
path.is_absolute()
|
||||
|| matches!(value, "." | "..")
|
||||
|| value.starts_with("./")
|
||||
|| value.starts_with("../")
|
||||
|| value.starts_with("~/")
|
||||
}
|
||||
|
||||
fn value_looks_like_bare_dockerfile_name(value: &str) -> bool {
|
||||
!value.contains('/') && !value.contains(':') && filename_looks_like_dockerfile(Path::new(value))
|
||||
}
|
||||
|
||||
fn source_requests_gpu(source: &str) -> bool {
|
||||
resolve_from(source).is_ok_and(|resolved| match resolved {
|
||||
ResolvedSource::Image(image) => image_requests_gpu(&image),
|
||||
@@ -2849,15 +2897,29 @@ fn image_requests_gpu(image: &str) -> bool {
|
||||
image_name.contains("gpu")
|
||||
}
|
||||
|
||||
/// Build a Dockerfile and push the resulting image into the gateway.
|
||||
fn dockerfile_sources_supported_for_gateway(metadata: Option<&GatewayMetadata>) -> bool {
|
||||
!metadata.is_some_and(|metadata| metadata.is_remote)
|
||||
}
|
||||
|
||||
/// Build a Dockerfile and make the resulting image available to the gateway.
|
||||
///
|
||||
/// Returns the image tag that was built so the caller can use it for sandbox
|
||||
/// creation.
|
||||
/// For local Kubernetes gateways running in Docker, this imports the built image
|
||||
/// into the gateway runtime and returns the Docker tag. Standalone local
|
||||
/// gateways use the same Docker daemon that the CLI built into, so the tag is
|
||||
/// passed through directly and the active compute driver resolves it.
|
||||
async fn build_from_dockerfile(
|
||||
dockerfile: &Path,
|
||||
context: &Path,
|
||||
gateway_name: &str,
|
||||
) -> Result<String> {
|
||||
let metadata = get_gateway_metadata(gateway_name);
|
||||
if !dockerfile_sources_supported_for_gateway(metadata.as_ref()) {
|
||||
return Err(miette!(
|
||||
"local Dockerfile sources are only supported for local gateways; gateway '{}' is remote",
|
||||
gateway_name
|
||||
));
|
||||
}
|
||||
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
@@ -2886,11 +2948,30 @@ async fn build_from_dockerfile(
|
||||
)
|
||||
.await?;
|
||||
|
||||
let existing_gateway = openshell_bootstrap::check_existing_deployment(gateway_name, None)
|
||||
.await
|
||||
.wrap_err("failed to inspect local gateway deployment state")?;
|
||||
let pushed_into_gateway = existing_gateway
|
||||
.is_some_and(|gateway| gateway.container_exists && gateway.container_running);
|
||||
if pushed_into_gateway {
|
||||
openshell_bootstrap::build::push_image_into_gateway(&tag, gateway_name, &mut on_log)
|
||||
.await?;
|
||||
eprintln!();
|
||||
eprintln!(
|
||||
"{} Image {} is available in the gateway.",
|
||||
"✓".green().bold(),
|
||||
tag.cyan(),
|
||||
);
|
||||
eprintln!();
|
||||
return Ok(tag);
|
||||
}
|
||||
|
||||
eprintln!();
|
||||
eprintln!(
|
||||
"{} Image {} is available in the local Docker daemon.",
|
||||
"{} Image {} is available in the local Docker daemon for gateway '{}'.",
|
||||
"✓".green().bold(),
|
||||
tag.cyan(),
|
||||
gateway_name,
|
||||
);
|
||||
eprintln!();
|
||||
|
||||
@@ -5711,13 +5792,14 @@ fn format_timestamp_ms(ms: i64) -> String {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
GatewayControlTarget, TlsOptions, format_gateway_select_header,
|
||||
format_gateway_select_items, gateway_add, gateway_auth_label, gateway_select_with,
|
||||
gateway_type_label, git_sync_files, http_health_check, image_requests_gpu,
|
||||
inferred_provider_type, parse_cli_setting_value, parse_credential_pairs,
|
||||
plaintext_gateway_is_remote, provisioning_timeout_message, ready_false_condition_message,
|
||||
resolve_gateway_control_target_from, sandbox_should_persist, shell_escape,
|
||||
source_requests_gpu, validate_gateway_name, validate_ssh_host,
|
||||
GatewayControlTarget, TlsOptions, dockerfile_sources_supported_for_gateway,
|
||||
format_gateway_select_header, format_gateway_select_items, gateway_add, gateway_auth_label,
|
||||
gateway_select_with, gateway_type_label, git_sync_files, http_health_check,
|
||||
image_requests_gpu, inferred_provider_type, parse_cli_setting_value,
|
||||
parse_credential_pairs, plaintext_gateway_is_remote, provisioning_timeout_message,
|
||||
ready_false_condition_message, resolve_from, resolve_gateway_control_target_from,
|
||||
sandbox_should_persist, shell_escape, source_requests_gpu, validate_gateway_name,
|
||||
validate_ssh_host,
|
||||
};
|
||||
use crate::TEST_ENV_LOCK;
|
||||
use hyper::StatusCode;
|
||||
@@ -5963,6 +6045,103 @@ mod tests {
|
||||
assert!(!source_requests_gpu("base"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_from_classifies_existing_dockerfile_path() {
|
||||
let temp = tempfile::tempdir().expect("failed to create tempdir");
|
||||
let dockerfile = temp.path().join("Dockerfile");
|
||||
fs::write(&dockerfile, "FROM scratch\n").expect("failed to write Dockerfile");
|
||||
|
||||
match resolve_from(dockerfile.to_str().expect("temp path is not UTF-8"))
|
||||
.expect("expected Dockerfile source")
|
||||
{
|
||||
super::ResolvedSource::Dockerfile {
|
||||
dockerfile: resolved,
|
||||
context,
|
||||
} => {
|
||||
assert_eq!(
|
||||
resolved,
|
||||
dockerfile
|
||||
.canonicalize()
|
||||
.expect("failed to canonicalize Dockerfile")
|
||||
);
|
||||
assert_eq!(
|
||||
context,
|
||||
temp.path()
|
||||
.canonicalize()
|
||||
.expect("failed to canonicalize context")
|
||||
);
|
||||
}
|
||||
super::ResolvedSource::Image(image) => {
|
||||
panic!("expected Dockerfile source, got image {image}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_from_rejects_missing_explicit_dockerfile_path() {
|
||||
let temp = tempfile::tempdir().expect("failed to create tempdir");
|
||||
let missing = temp.path().join("Dockerfile");
|
||||
|
||||
let err = resolve_from(missing.to_str().expect("temp path is not UTF-8"))
|
||||
.expect_err("expected missing Dockerfile path to be rejected");
|
||||
|
||||
assert!(
|
||||
err.to_string().contains("local --from path does not exist"),
|
||||
"unexpected error: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_from_keeps_dockerfile_named_image_refs_as_images() {
|
||||
let image_ref = "ghcr.io/acme/dockerfile-runner:latest";
|
||||
|
||||
match resolve_from(image_ref).expect("expected image source") {
|
||||
super::ResolvedSource::Image(image) => assert_eq!(image, image_ref),
|
||||
super::ResolvedSource::Dockerfile { .. } => {
|
||||
panic!("expected image ref, got Dockerfile source");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dockerfile_sources_are_rejected_for_remote_gateways() {
|
||||
let metadata = GatewayMetadata {
|
||||
name: "remote".to_string(),
|
||||
gateway_endpoint: "https://gateway.example.com".to_string(),
|
||||
is_remote: true,
|
||||
gateway_port: 443,
|
||||
remote_host: Some("user@gateway.example.com".to_string()),
|
||||
resolved_host: Some("gateway.example.com".to_string()),
|
||||
auth_mode: None,
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(!dockerfile_sources_supported_for_gateway(Some(&metadata)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dockerfile_sources_are_allowed_for_local_gateways() {
|
||||
let metadata = GatewayMetadata {
|
||||
name: "local".to_string(),
|
||||
gateway_endpoint: "http://127.0.0.1:8080".to_string(),
|
||||
is_remote: false,
|
||||
gateway_port: 8080,
|
||||
remote_host: None,
|
||||
resolved_host: None,
|
||||
auth_mode: None,
|
||||
edge_team_domain: None,
|
||||
edge_auth_url: None,
|
||||
vm_driver_state_dir: None,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(dockerfile_sources_supported_for_gateway(Some(&metadata)));
|
||||
assert!(dockerfile_sources_supported_for_gateway(None));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ready_false_condition_message_prefers_reason_and_message() {
|
||||
let status = SandboxStatus {
|
||||
@@ -6301,6 +6480,7 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_health_check_supports_plain_http_endpoints() {
|
||||
let _ = rustls::crypto::ring::default_provider().install_default();
|
||||
let listener = TcpListener::bind("127.0.0.1:0").expect("bind listener");
|
||||
let addr = listener.local_addr().expect("listener addr");
|
||||
let server = thread::spawn(move || {
|
||||
|
||||
@@ -735,6 +735,10 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() {
|
||||
let _env = test_env(&fake_ssh_dir, &xdg_dir);
|
||||
let tls = test_tls(&server);
|
||||
install_fake_ssh(&fake_ssh_dir);
|
||||
let forward_port = {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
listener.local_addr().unwrap().port()
|
||||
};
|
||||
|
||||
run::sandbox_create(
|
||||
&server.endpoint,
|
||||
@@ -750,7 +754,7 @@ async fn sandbox_create_keeps_sandbox_with_forwarding() {
|
||||
None,
|
||||
&[],
|
||||
None,
|
||||
Some(openshell_core::forward::ForwardSpec::new(8080)),
|
||||
Some(openshell_core::forward::ForwardSpec::new(forward_port)),
|
||||
&["echo".to_string(), "OK".to_string()],
|
||||
Some(false),
|
||||
Some(false),
|
||||
|
||||
@@ -1019,12 +1019,28 @@ mod tests {
|
||||
#[test]
|
||||
fn check_port_available_free_port() {
|
||||
// Bind to port 0 to get an OS-assigned free port, then drop the
|
||||
// listener so the port is released before we test it.
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
drop(listener);
|
||||
// listener so the port is released before we test it. On busy CI
|
||||
// hosts, another process can claim that single ephemeral port before
|
||||
// we re-bind it, so retry with fresh OS-assigned ports.
|
||||
let mut last_error = None;
|
||||
for _ in 0..20 {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
drop(listener);
|
||||
|
||||
assert!(check_port_available(&ForwardSpec::new(port)).is_ok());
|
||||
match check_port_available(&ForwardSpec::new(port)) {
|
||||
Ok(()) => return,
|
||||
Err(err) => {
|
||||
last_error = Some(err.to_string());
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
panic!(
|
||||
"expected an OS-assigned port to be available; last error: {}",
|
||||
last_error.unwrap_or_else(|| "none".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -22,6 +22,7 @@ path = "src/main.rs"
|
||||
openshell-core = { path = "../openshell-core" }
|
||||
openshell-vfio = { path = "../openshell-vfio" }
|
||||
|
||||
bollard = { version = "0.20", features = ["ssh"] }
|
||||
tokio = { workspace = true }
|
||||
tonic = { workspace = true, features = ["transport"] }
|
||||
prost-types = { workspace = true }
|
||||
@@ -35,9 +36,12 @@ miette = { workspace = true }
|
||||
url = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
oci-client = "0.16"
|
||||
libc = "0.2"
|
||||
libloading = "0.8"
|
||||
tar = "0.4"
|
||||
flate2 = "1"
|
||||
sha2 = "0.10"
|
||||
zstd = "0.13"
|
||||
|
||||
# smol-rs/polling drives the BSD/macOS parent-death detection in
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
> Status: Experimental. The VM compute driver is under active development and the interface still has VM-specific plumbing that will be generalized.
|
||||
|
||||
Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess, talks to it over a Unix domain socket with the `openshell.compute.v1.ComputeDriver` gRPC surface, and lets it manage per-sandbox microVMs. The runtime (libkrun + libkrunfw + gvproxy) and sandbox rootfs are embedded directly in the binary — no sibling files required at runtime.
|
||||
Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) for OpenShell. The gateway spawns this binary as a subprocess, talks to it over a Unix domain socket with the `openshell.compute.v1.ComputeDriver` gRPC surface, and lets it manage per-sandbox microVMs. The runtime (libkrun + libkrunfw + gvproxy) and the sandbox supervisor are embedded directly in the binary; each sandbox guest rootfs is derived from a configured container image at create time.
|
||||
|
||||
## How it fits together
|
||||
|
||||
@@ -10,7 +10,7 @@ Standalone libkrun-backed [`ComputeDriver`](../../proto/compute_driver.proto) fo
|
||||
flowchart LR
|
||||
subgraph host["Host process"]
|
||||
gateway["openshell-server<br/>(compute::vm::spawn)"]
|
||||
driver["openshell-driver-vm<br/>├── libkrun (VM)<br/>├── gvproxy (net)<br/>└── rootfs.tar.zst"]
|
||||
driver["openshell-driver-vm<br/>├── libkrun (VM)<br/>├── gvproxy (net)<br/>└── openshell-sandbox.zst"]
|
||||
gateway <-->|"gRPC over UDS<br/>compute-driver.sock"| driver
|
||||
end
|
||||
|
||||
@@ -35,9 +35,15 @@ Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside th
|
||||
mise run gateway:vm
|
||||
```
|
||||
|
||||
First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:rootfs -- --base` builds the embedded rootfs. Subsequent runs are cached. To keep the Unix socket path under macOS `SUN_LEN`, `mise run gateway:vm` and `start.sh` default the state dir to `/tmp/openshell-vm-driver-dev-$USER-port-$PORT/` (SQLite DB + per-sandbox rootfs + `compute-driver.sock`) unless `OPENSHELL_VM_DRIVER_STATE_DIR` is set.
|
||||
The wrapper auto-registers the gateway with the CLI (`gateway destroy` + `gateway add`) so no manual registration step is needed. When running under `sudo`, it uses `sudo -u $SUDO_USER` for the registration so the config is written under the invoking user's home directory. Re-runs are idempotent.
|
||||
It also exports `OPENSHELL_DRIVER_DIR=$PWD/target/debug` before starting the gateway so local dev runs use the freshly built `openshell-driver-vm` instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`.
|
||||
First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:supervisor` builds the bundled guest supervisor. Subsequent runs are cached.
|
||||
|
||||
By default `mise run gateway:vm`:
|
||||
|
||||
- Listens on plaintext HTTP at `127.0.0.1:18081`.
|
||||
- Registers the CLI gateway `vm-dev` by writing `~/.config/openshell/gateways/vm-dev/metadata.json`. It does not modify the workspace `.env`.
|
||||
- Persists the gateway SQLite DB under `.cache/gateway-vm/gateway.db`.
|
||||
- Places the VM driver state (per-sandbox rootfs + `compute-driver.sock`) under `/tmp/openshell-vm-driver-$USER-vm-dev/` so the AF_UNIX socket path stays under macOS `SUN_LEN`.
|
||||
- Passes `--driver-dir $PWD/target/debug` so the freshly built `openshell-driver-vm` is used instead of an older installed copy from `~/.local/libexec/openshell`, `/usr/libexec/openshell`, or `/usr/local/libexec`.
|
||||
|
||||
For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges:
|
||||
|
||||
@@ -47,50 +53,43 @@ sudo -E env "PATH=$PATH" mise run gateway:vm -- --gpu
|
||||
|
||||
See [`architecture/vm-gpu-sandbox-guide.md`](../../architecture/vm-gpu-sandbox-guide.md) for full GPU prerequisites and usage.
|
||||
|
||||
Override via environment:
|
||||
Point the CLI at the gateway with one of:
|
||||
|
||||
```shell
|
||||
OPENSHELL_SERVER_PORT=9090 \
|
||||
crates/openshell-driver-vm/start.sh
|
||||
openshell --gateway vm-dev status
|
||||
openshell gateway select vm-dev # then plain `openshell <command>`
|
||||
```
|
||||
|
||||
Run multiple dev gateways side by side by giving each one a unique port. The wrapper derives a distinct default state dir from that port automatically:
|
||||
Override defaults via environment:
|
||||
|
||||
```shell
|
||||
OPENSHELL_SERVER_PORT=8080 mise run gateway:vm
|
||||
OPENSHELL_SERVER_PORT=8081 mise run gateway:vm
|
||||
```
|
||||
# custom port (fails fast if in use)
|
||||
OPENSHELL_SERVER_PORT=18091 mise run gateway:vm
|
||||
|
||||
If you want a custom suffix instead of `port-$PORT`, set `OPENSHELL_VM_INSTANCE`:
|
||||
|
||||
```shell
|
||||
OPENSHELL_SERVER_PORT=8082 \
|
||||
OPENSHELL_VM_INSTANCE=feature-a \
|
||||
mise run gateway:vm
|
||||
```
|
||||
|
||||
If you want a custom CLI gateway name, set `OPENSHELL_VM_GATEWAY_NAME`:
|
||||
|
||||
```shell
|
||||
OPENSHELL_SERVER_PORT=8082 \
|
||||
# custom CLI gateway name + namespace
|
||||
OPENSHELL_VM_GATEWAY_NAME=vm-feature-a \
|
||||
OPENSHELL_SANDBOX_NAMESPACE=vm-feature-a \
|
||||
mise run gateway:vm
|
||||
|
||||
# custom sandbox image
|
||||
OPENSHELL_SANDBOX_IMAGE=ghcr.io/example/sandbox:latest mise run gateway:vm
|
||||
```
|
||||
|
||||
Teardown:
|
||||
|
||||
```shell
|
||||
rm -rf /tmp/openshell-vm-driver-dev-$USER-port-8080
|
||||
rm -rf /tmp/openshell-vm-driver-$USER-vm-dev .cache/gateway-vm
|
||||
rm -rf "${XDG_CONFIG_HOME:-$HOME/.config}/openshell/gateways/vm-dev"
|
||||
```
|
||||
|
||||
## Manual equivalent
|
||||
|
||||
If you want to drive the launch yourself instead of using `start.sh`:
|
||||
If you want to drive the launch yourself instead of using `mise run gateway:vm` (i.e. `tasks/scripts/gateway-vm.sh`):
|
||||
|
||||
```shell
|
||||
# 1. Stage runtime artifacts + base rootfs into target/vm-runtime-compressed/
|
||||
# 1. Stage runtime artifacts + supervisor bundle into target/vm-runtime-compressed/
|
||||
mise run vm:setup
|
||||
mise run vm:rootfs -- --base # if rootfs.tar.zst is not already present
|
||||
mise run vm:supervisor # if openshell-sandbox.zst is not already present
|
||||
|
||||
# 2. Build both binaries with the staged artifacts embedded
|
||||
OPENSHELL_VM_RUNTIME_COMPRESSED_DIR=$PWD/target/vm-runtime-compressed \
|
||||
@@ -102,16 +101,17 @@ codesign \
|
||||
--force -s - target/debug/openshell-driver-vm
|
||||
|
||||
# 4. Start the gateway with the VM driver
|
||||
mkdir -p /tmp/openshell-vm-driver-dev-$USER-port-8080
|
||||
mkdir -p /tmp/openshell-vm-driver-$USER-vm-dev .cache/gateway-vm
|
||||
target/debug/openshell-gateway \
|
||||
--drivers vm \
|
||||
--disable-tls \
|
||||
--database-url sqlite:/tmp/openshell-vm-driver-dev-$USER-port-8080/openshell.db \
|
||||
--db-url "sqlite:.cache/gateway-vm/gateway.db?mode=rwc" \
|
||||
--driver-dir $PWD/target/debug \
|
||||
--grpc-endpoint http://host.containers.internal:8080 \
|
||||
--ssh-gateway-host 127.0.0.1 \
|
||||
--ssh-gateway-port 8080 \
|
||||
--vm-driver-state-dir /tmp/openshell-vm-driver-dev-$USER-port-8080
|
||||
--sandbox-namespace vm-dev \
|
||||
--sandbox-image <compatible-image> \
|
||||
--grpc-endpoint http://host.containers.internal:18081 \
|
||||
--port 18081 \
|
||||
--vm-driver-state-dir /tmp/openshell-vm-driver-$USER-vm-dev
|
||||
```
|
||||
|
||||
The gateway resolves `openshell-driver-vm` in this order: `--driver-dir`, conventional install locations (`~/.local/libexec/openshell`, `/usr/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`), then a sibling of the gateway binary.
|
||||
@@ -121,7 +121,7 @@ The gateway resolves `openshell-driver-vm` in this order: `--driver-dir`, conven
|
||||
| Flag | Env var | Default | Purpose |
|
||||
|---|---|---|---|
|
||||
| `--drivers vm` | `OPENSHELL_DRIVERS` | `kubernetes` | Select the VM compute driver. |
|
||||
| `--grpc-endpoint URL` | `OPENSHELL_GRPC_ENDPOINT` | — | Required. URL the sandbox guest calls back to. Use a host alias that resolves to the gateway's host from inside the VM (`host.containers.internal` comes from gvproxy DNS; the guest init script also seeds `host.openshell.internal` to `192.168.127.1`). |
|
||||
| `--grpc-endpoint URL` | `OPENSHELL_GRPC_ENDPOINT` | — | Required. URL the sandbox guest dials to reach the gateway. Use `http://host.containers.internal:<port>` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:<port>` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. |
|
||||
| `--vm-driver-state-dir DIR` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Per-sandbox rootfs, console logs, and the `compute-driver.sock` UDS. |
|
||||
| `--driver-dir DIR` | `OPENSHELL_DRIVER_DIR` | unset | Override the directory searched for `openshell-driver-vm`. |
|
||||
| `--vm-driver-vcpus N` | `OPENSHELL_VM_DRIVER_VCPUS` | `2` | vCPUs per sandbox. |
|
||||
@@ -135,14 +135,15 @@ See [`openshell-gateway --help`](../openshell-server/src/cli.rs) for the full fl
|
||||
|
||||
## Verifying the gateway
|
||||
|
||||
The gateway is auto-registered by `start.sh`. In another terminal:
|
||||
The gateway is auto-registered by `mise run gateway:vm`. In another terminal:
|
||||
|
||||
```shell
|
||||
scripts/bin/openshell sandbox create --name demo
|
||||
scripts/bin/openshell sandbox connect demo
|
||||
./scripts/bin/openshell status
|
||||
./scripts/bin/openshell sandbox create --name demo --from <compatible-image>
|
||||
./scripts/bin/openshell sandbox connect demo
|
||||
```
|
||||
|
||||
First sandbox takes 10–30 seconds to boot (rootfs extraction + libkrun + guest init). Subsequent creates reuse the prepared sandbox rootfs.
|
||||
First sandbox takes 10–30 seconds to boot (image fetch/prepare/cache + libkrun + guest init). If `--from` is omitted, the VM driver uses the gateway's configured default sandbox image. Without either `--from` or `--sandbox-image`, VM sandbox creation fails. Subsequent creates reuse the prepared sandbox rootfs.
|
||||
|
||||
## Logs and debugging
|
||||
|
||||
@@ -150,7 +151,7 @@ Raise log verbosity for both processes:
|
||||
|
||||
```shell
|
||||
RUST_LOG=openshell_server=debug,openshell_driver_vm=debug \
|
||||
crates/openshell-driver-vm/start.sh
|
||||
mise run gateway:vm
|
||||
```
|
||||
|
||||
The VM guest's serial console is appended to `<state-dir>/<sandbox-id>/console.log`. The `compute-driver.sock` lives at `<state-dir>/compute-driver.sock`; the gateway removes it on clean shutdown via `ManagedDriverProcess::drop`.
|
||||
@@ -161,9 +162,11 @@ The VM guest's serial console is appended to `<state-dir>/<sandbox-id>/console.l
|
||||
- Rust toolchain
|
||||
- Guest-supervisor cross-compile toolchain (needed on macOS, and on Linux when host arch ≠ guest arch):
|
||||
- Matching rustup target: `rustup target add aarch64-unknown-linux-gnu` (or `x86_64-unknown-linux-gnu` for an amd64 guest)
|
||||
- `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `build-rootfs.sh` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary.
|
||||
- `cargo install --locked cargo-zigbuild` and `brew install zig` (or distro equivalent). `vm:supervisor` uses `cargo zigbuild` to cross-compile the in-VM `openshell-sandbox` supervisor binary.
|
||||
- [mise](https://mise.jdx.dev/) task runner
|
||||
- Docker (needed by `mise run vm:rootfs` to build the base rootfs)
|
||||
- Docker-compatible socket on the local CLI/gateway host when using
|
||||
`openshell sandbox create --from ./Dockerfile` or `--from ./dir`; the CLI
|
||||
builds the image and the VM driver exports it via the local Docker daemon
|
||||
- `gh` CLI (used by `mise run vm:setup` to download pre-built runtime artifacts)
|
||||
|
||||
## Relationship to `openshell-vm`
|
||||
@@ -173,4 +176,4 @@ The VM guest's serial console is appended to `<state-dir>/<sandbox-id>/console.l
|
||||
## TODOs
|
||||
|
||||
- The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` notes in `crates/openshell-server/src/lib.rs` and `crates/openshell-server/src/compute/vm.rs`.
|
||||
- macOS codesigning is handled by `start.sh`; a packaged release would need signing in CI.
|
||||
- macOS codesigning is handled by `tasks/scripts/gateway-vm.sh`; a packaged release would need signing in CI.
|
||||
|
||||
@@ -3,9 +3,9 @@
|
||||
|
||||
//! Build script for openshell-driver-vm.
|
||||
//!
|
||||
//! This crate embeds the sandbox rootfs plus the minimal libkrun runtime
|
||||
//! artifacts it needs to boot base VMs without depending on the openshell-vm
|
||||
//! binary or crate.
|
||||
//! This crate embeds the sandbox supervisor plus the minimal libkrun runtime
|
||||
//! artifacts it needs to boot VMs without depending on the openshell-vm binary
|
||||
//! or crate.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::{env, fs};
|
||||
@@ -21,8 +21,7 @@ fn main() {
|
||||
"libkrun.dylib.zst",
|
||||
"libkrunfw.5.dylib.zst",
|
||||
"gvproxy.zst",
|
||||
"rootfs.tar.zst",
|
||||
"rootfs-gpu.tar.zst",
|
||||
"openshell-sandbox.zst",
|
||||
] {
|
||||
println!("cargo:rerun-if-changed={dir}/{name}");
|
||||
}
|
||||
@@ -37,15 +36,7 @@ fn main() {
|
||||
"linux" => ("libkrun.so", "libkrunfw.so.5"),
|
||||
_ => {
|
||||
println!("cargo:warning=VM runtime not available for {target_os}-{target_arch}");
|
||||
generate_stub_resources(
|
||||
&out_dir,
|
||||
&[
|
||||
"libkrun",
|
||||
"libkrunfw",
|
||||
"rootfs.tar.zst",
|
||||
"rootfs-gpu.tar.zst",
|
||||
],
|
||||
);
|
||||
generate_stub_resources(&out_dir, &["libkrun", "libkrunfw", "openshell-sandbox.zst"]);
|
||||
return;
|
||||
}
|
||||
};
|
||||
@@ -54,15 +45,14 @@ fn main() {
|
||||
PathBuf::from(dir)
|
||||
} else {
|
||||
println!("cargo:warning=OPENSHELL_VM_RUNTIME_COMPRESSED_DIR not set");
|
||||
println!("cargo:warning=Run: mise run vm:setup");
|
||||
println!("cargo:warning=Run: mise run vm:setup && mise run vm:supervisor");
|
||||
generate_stub_resources(
|
||||
&out_dir,
|
||||
&[
|
||||
&format!("{libkrun_name}.zst"),
|
||||
&format!("{libkrunfw_name}.zst"),
|
||||
"gvproxy.zst",
|
||||
"rootfs.tar.zst",
|
||||
"rootfs-gpu.tar.zst",
|
||||
"openshell-sandbox.zst",
|
||||
],
|
||||
);
|
||||
return;
|
||||
@@ -73,15 +63,14 @@ fn main() {
|
||||
"cargo:warning=Compressed runtime dir not found: {}",
|
||||
compressed_dir.display()
|
||||
);
|
||||
println!("cargo:warning=Run: mise run vm:setup");
|
||||
println!("cargo:warning=Run: mise run vm:setup && mise run vm:supervisor");
|
||||
generate_stub_resources(
|
||||
&out_dir,
|
||||
&[
|
||||
&format!("{libkrun_name}.zst"),
|
||||
&format!("{libkrunfw_name}.zst"),
|
||||
"gvproxy.zst",
|
||||
"rootfs.tar.zst",
|
||||
"rootfs-gpu.tar.zst",
|
||||
"openshell-sandbox.zst",
|
||||
],
|
||||
);
|
||||
return;
|
||||
@@ -94,10 +83,9 @@ fn main() {
|
||||
format!("{libkrunfw_name}.zst"),
|
||||
),
|
||||
("gvproxy.zst".to_string(), "gvproxy.zst".to_string()),
|
||||
("rootfs.tar.zst".to_string(), "rootfs.tar.zst".to_string()),
|
||||
(
|
||||
"rootfs-gpu.tar.zst".to_string(),
|
||||
"rootfs-gpu.tar.zst".to_string(),
|
||||
"openshell-sandbox.zst".to_string(),
|
||||
"openshell-sandbox.zst".to_string(),
|
||||
),
|
||||
];
|
||||
|
||||
@@ -131,15 +119,16 @@ fn main() {
|
||||
}
|
||||
|
||||
if !all_found {
|
||||
println!("cargo:warning=Some artifacts missing. Run: mise run vm:setup");
|
||||
println!(
|
||||
"cargo:warning=Some artifacts missing. Run: mise run vm:setup && mise run vm:supervisor"
|
||||
);
|
||||
generate_stub_resources(
|
||||
&out_dir,
|
||||
&[
|
||||
&format!("{libkrun_name}.zst"),
|
||||
&format!("{libkrunfw_name}.zst"),
|
||||
"gvproxy.zst",
|
||||
"rootfs.tar.zst",
|
||||
"rootfs-gpu.tar.zst",
|
||||
"openshell-sandbox.zst",
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
@@ -9,30 +9,34 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Source QEMU-injected environment variables if present
|
||||
# Source QEMU-injected environment variables if present.
|
||||
if [ -f /srv/openshell-env.sh ]; then
|
||||
# shellcheck source=/dev/null
|
||||
source /srv/openshell-env.sh
|
||||
fi
|
||||
|
||||
BOOT_START=$(date +%s%3N 2>/dev/null || date +%s)
|
||||
# gvisor-tap-vsock subnet layout:
|
||||
# 192.168.127.1 — gateway: gvproxy's DNS / DHCP / HTTP API. Does NOT
|
||||
# proxy arbitrary host ports.
|
||||
# 192.168.127.254 — host-loopback: NAT-rewritten to host's 127.0.0.1 by
|
||||
# gvproxy's TCP/UDP/ICMP forwarder. Use this address
|
||||
# (or any of the host.* hostnames below) to reach a
|
||||
# service the host is listening on.
|
||||
# The host.containers.internal / host.docker.internal DNS records served
|
||||
# by gvproxy's embedded resolver point at 192.168.127.254. We mirror that
|
||||
# in /etc/hosts so the supervisor can reach the gateway even when
|
||||
# gvproxy's DNS is not in resolv.conf (e.g. DHCP failed and we fell
|
||||
# back to 8.8.8.8).
|
||||
GVPROXY_GATEWAY_IP="192.168.127.1"
|
||||
GVPROXY_HOST_LOOPBACK_IP="192.168.127.254"
|
||||
GATEWAY_IP="$GVPROXY_GATEWAY_IP"
|
||||
|
||||
# Parse kernel cmdline for GPU and TAP networking parameters
|
||||
GPU_ENABLED="${GPU_ENABLED:-false}"
|
||||
VM_NET_IP="${VM_NET_IP:-}"
|
||||
VM_NET_GW="${VM_NET_GW:-}"
|
||||
VM_NET_DNS="${VM_NET_DNS:-}"
|
||||
|
||||
for param in $(cat /proc/cmdline 2>/dev/null || true); do
|
||||
case "$param" in
|
||||
GPU_ENABLED=*) GPU_ENABLED="${param#GPU_ENABLED=}" ;;
|
||||
VM_NET_IP=*) VM_NET_IP="${param#VM_NET_IP=}" ;;
|
||||
VM_NET_GW=*) VM_NET_GW="${param#VM_NET_GW=}" ;;
|
||||
VM_NET_DNS=*) VM_NET_DNS="${param#VM_NET_DNS=}" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
ts() {
|
||||
local now
|
||||
now=$(date +%s%3N 2>/dev/null || date +%s)
|
||||
@@ -89,22 +93,46 @@ tcp_probe() {
|
||||
local port="$2"
|
||||
|
||||
if command -v timeout >/dev/null 2>&1; then
|
||||
timeout 2 bash -c "exec 3<>/dev/tcp/${host}/${port}" >/dev/null 2>&1
|
||||
timeout 2 bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1
|
||||
else
|
||||
bash -c "exec 3<>/dev/tcp/${host}/${port}" >/dev/null 2>&1
|
||||
bash -c "exec 3<>/dev/tcp/\$1/\$2" _ "$host" "$port" >/dev/null 2>&1
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_host_gateway_aliases() {
|
||||
# Seed /etc/hosts with the well-known gvproxy hostnames so the supervisor
|
||||
# can reach the OpenShell server even when gvproxy's built-in DNS is not
|
||||
# in resolv.conf (e.g. when DHCP fails and we fall back to 8.8.8.8).
|
||||
#
|
||||
# Critical distinction: host.* aliases point at the gvproxy *host-loopback*
|
||||
# IP (192.168.127.254), not the gateway IP (192.168.127.1). Only the
|
||||
# host-loopback IP carries NAT rewriting to the host's 127.0.0.1 — the
|
||||
# gateway IP only listens on gvproxy's own service ports (DNS:53, DHCP,
|
||||
# HTTP API:80). Pinning host.containers.internal to the gateway IP
|
||||
# silently breaks guest→host port reachability for arbitrary ports.
|
||||
local hosts_tmp="/tmp/openshell-hosts.$$"
|
||||
local host_aliases="host.openshell.internal host.containers.internal host.docker.internal"
|
||||
local gateway_aliases="gateway.containers.internal"
|
||||
local filter='(^|[[:space:]])(host\.openshell\.internal|host\.containers\.internal|host\.docker\.internal|gateway\.containers\.internal)([[:space:]]|$)'
|
||||
|
||||
if [ -f /etc/hosts ]; then
|
||||
grep -vE '(^|[[:space:]])host\.openshell\.internal([[:space:]]|$)' /etc/hosts > "$hosts_tmp" || true
|
||||
grep -vE "$filter" /etc/hosts > "$hosts_tmp" || true
|
||||
else
|
||||
: > "$hosts_tmp"
|
||||
fi
|
||||
|
||||
printf '%s host.openshell.internal\n' "$GATEWAY_IP" >> "$hosts_tmp"
|
||||
# In TAP/GPU mode, GATEWAY_IP is overridden to VM_NET_GW (the host-side
|
||||
# of the TAP), and the gateway is reachable directly there. In gvproxy
|
||||
# mode, host.openshell.internal etc. need GVPROXY_HOST_LOOPBACK_IP
|
||||
# (192.168.127.254) which is gvproxy's host-NAT entry, while
|
||||
# gateway.containers.internal points at the gvproxy gateway itself.
|
||||
if [ "${GATEWAY_IP}" = "${GVPROXY_GATEWAY_IP}" ]; then
|
||||
printf '%s %s\n' "$GVPROXY_HOST_LOOPBACK_IP" "$host_aliases" >> "$hosts_tmp"
|
||||
printf '%s %s\n' "$GVPROXY_GATEWAY_IP" "$gateway_aliases" >> "$hosts_tmp"
|
||||
else
|
||||
# TAP networking: gateway and host are both reachable at GATEWAY_IP.
|
||||
printf '%s %s %s\n' "$GATEWAY_IP" "$host_aliases" "$gateway_aliases" >> "$hosts_tmp"
|
||||
fi
|
||||
cat "$hosts_tmp" > /etc/hosts
|
||||
rm -f "$hosts_tmp"
|
||||
}
|
||||
@@ -129,7 +157,15 @@ rewrite_openshell_endpoint_if_needed() {
|
||||
return 0
|
||||
fi
|
||||
|
||||
for candidate in host.openshell.internal host.containers.internal host.docker.internal "$GATEWAY_IP"; do
|
||||
# Probe candidates in preference order. Hostnames first for informative
|
||||
# log output, then a bare IP as a final safety net. In gvproxy mode the
|
||||
# bare IP is the host-loopback (192.168.127.254). In TAP/GPU mode it's
|
||||
# the TAP host gateway.
|
||||
local fallback_ip="$GVPROXY_HOST_LOOPBACK_IP"
|
||||
if [ "${GATEWAY_IP}" != "${GVPROXY_GATEWAY_IP}" ]; then
|
||||
fallback_ip="$GATEWAY_IP"
|
||||
fi
|
||||
for candidate in host.openshell.internal host.containers.internal host.docker.internal "$fallback_ip"; do
|
||||
if [ "$candidate" = "$host" ]; then
|
||||
continue
|
||||
fi
|
||||
@@ -244,16 +280,12 @@ mount -t tmpfs tmpfs /run 2>/dev/null &
|
||||
mount -t devtmpfs devtmpfs /dev 2>/dev/null &
|
||||
wait
|
||||
|
||||
mkdir -p /dev/pts /dev/shm /sys/fs/cgroup /sandbox
|
||||
mkdir -p /dev/pts /dev/shm /sys/fs/cgroup
|
||||
mount -t devpts devpts /dev/pts 2>/dev/null &
|
||||
mount -t tmpfs tmpfs /dev/shm 2>/dev/null &
|
||||
mount -t cgroup2 cgroup2 /sys/fs/cgroup 2>/dev/null &
|
||||
wait
|
||||
|
||||
mount -t tmpfs tmpfs /sandbox 2>/dev/null || true
|
||||
mkdir -p /sandbox
|
||||
chown sandbox:sandbox /sandbox 2>/dev/null || true
|
||||
|
||||
hostname openshell-sandbox-vm 2>/dev/null || true
|
||||
ip link set lo up 2>/dev/null || true
|
||||
|
||||
@@ -271,12 +303,22 @@ if [ -n "${VM_NET_IP}" ] && [ -n "${VM_NET_GW}" ]; then
|
||||
TAP_NIC=""
|
||||
NIC_WAIT=0
|
||||
while [ -z "$TAP_NIC" ] && [ "$NIC_WAIT" -lt 10 ]; do
|
||||
for candidate in eth0 ens3 enp0s2 $(ls /sys/class/net/ 2>/dev/null | grep -v '^lo$'); do
|
||||
for candidate in eth0 ens3 enp0s2; do
|
||||
if ip link show "$candidate" >/dev/null 2>&1 && [ "$candidate" != "lo" ]; then
|
||||
TAP_NIC="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$TAP_NIC" ]; then
|
||||
for sys_nic in /sys/class/net/*; do
|
||||
[ -e "$sys_nic" ] || continue
|
||||
candidate="${sys_nic##*/}"
|
||||
if ip link show "$candidate" >/dev/null 2>&1 && [ "$candidate" != "lo" ]; then
|
||||
TAP_NIC="$candidate"
|
||||
break
|
||||
fi
|
||||
done
|
||||
fi
|
||||
if [ -z "$TAP_NIC" ]; then
|
||||
sleep 1
|
||||
NIC_WAIT=$((NIC_WAIT + 1))
|
||||
@@ -307,7 +349,7 @@ elif ip link show eth0 >/dev/null 2>&1; then
|
||||
if command -v udhcpc >/dev/null 2>&1; then
|
||||
UDHCPC_SCRIPT="/usr/share/udhcpc/default.script"
|
||||
if [ ! -f "$UDHCPC_SCRIPT" ]; then
|
||||
mkdir -p /usr/share/udhcpc
|
||||
UDHCPC_SCRIPT="/run/openshell-udhcpc.script"
|
||||
cat > "$UDHCPC_SCRIPT" <<'DHCP_SCRIPT'
|
||||
#!/bin/sh
|
||||
case "$1" in
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -62,6 +62,9 @@ struct Args {
|
||||
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
|
||||
openshell_endpoint: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_IMAGE", default_value = "")]
|
||||
default_image: String,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_VM_DRIVER_STATE_DIR",
|
||||
@@ -169,6 +172,7 @@ async fn main() -> Result<()> {
|
||||
.ok_or_else(|| miette::miette!("OPENSHELL_GRPC_ENDPOINT is required"))?,
|
||||
state_dir: args.state_dir,
|
||||
launcher_bin: None,
|
||||
default_image: args.default_image,
|
||||
ssh_handshake_secret: args.ssh_handshake_secret.unwrap_or_default(),
|
||||
ssh_handshake_skew_secs: args.ssh_handshake_skew_secs,
|
||||
log_level: args.log_level,
|
||||
|
||||
@@ -2,74 +2,138 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use std::fs;
|
||||
use std::io::Cursor;
|
||||
use std::fs::File;
|
||||
use std::io::{BufWriter, Cursor};
|
||||
use std::path::Path;
|
||||
|
||||
const ROOTFS: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rootfs.tar.zst"));
|
||||
const ROOTFS_GPU: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/rootfs-gpu.tar.zst"));
|
||||
const SUPERVISOR: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/openshell-sandbox.zst"));
|
||||
const ROOTFS_VARIANT_MARKER: &str = ".openshell-rootfs-variant";
|
||||
const SANDBOX_GUEST_INIT_PATH: &str = "/srv/openshell-vm-sandbox-init.sh";
|
||||
const SANDBOX_SUPERVISOR_PATH: &str = "/opt/openshell/bin/openshell-sandbox";
|
||||
|
||||
pub const fn sandbox_guest_init_path() -> &'static str {
|
||||
SANDBOX_GUEST_INIT_PATH
|
||||
}
|
||||
|
||||
pub fn extract_sandbox_rootfs_to(dest: &Path) -> Result<(), String> {
|
||||
extract_variant(
|
||||
ROOTFS,
|
||||
"sandbox",
|
||||
"sandbox rootfs not embedded. Build openshell-driver-vm with OPENSHELL_VM_RUNTIME_COMPRESSED_DIR set or run `mise run vm:setup` first",
|
||||
dest,
|
||||
pub fn prepare_sandbox_rootfs_from_image_root(
|
||||
rootfs: &Path,
|
||||
image_identity: &str,
|
||||
) -> Result<(), String> {
|
||||
prepare_sandbox_rootfs(rootfs)?;
|
||||
validate_sandbox_rootfs(rootfs)?;
|
||||
fs::write(
|
||||
rootfs.join(ROOTFS_VARIANT_MARKER),
|
||||
format!("{}:image:{image_identity}\n", env!("CARGO_PKG_VERSION")),
|
||||
)
|
||||
.map_err(|e| format!("write rootfs variant marker: {e}"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn extract_gpu_sandbox_rootfs_to(dest: &Path) -> Result<(), String> {
|
||||
extract_variant(
|
||||
ROOTFS_GPU,
|
||||
"sandbox-gpu",
|
||||
"GPU sandbox rootfs not embedded. Build with `mise run vm:rootfs -- --gpu` first",
|
||||
dest,
|
||||
)
|
||||
}
|
||||
|
||||
fn extract_variant(blob: &[u8], variant: &str, empty_msg: &str, dest: &Path) -> Result<(), String> {
|
||||
if blob.is_empty() {
|
||||
return Err(empty_msg.to_string());
|
||||
}
|
||||
|
||||
let expected_marker = format!("{}:{variant}", env!("CARGO_PKG_VERSION"));
|
||||
let marker_path = dest.join(ROOTFS_VARIANT_MARKER);
|
||||
|
||||
if dest.is_dir()
|
||||
&& fs::read_to_string(&marker_path).is_ok_and(|value| value.trim() == expected_marker)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
pub fn extract_rootfs_archive_to(archive_path: &Path, dest: &Path) -> Result<(), String> {
|
||||
if dest.exists() {
|
||||
fs::remove_dir_all(dest)
|
||||
.map_err(|e| format!("remove old rootfs {}: {e}", dest.display()))?;
|
||||
}
|
||||
|
||||
unpack_zstd_tar(blob, variant, dest)?;
|
||||
prepare_sandbox_rootfs(dest)?;
|
||||
fs::write(marker_path, format!("{expected_marker}\n"))
|
||||
.map_err(|e| format!("write rootfs variant marker: {e}"))?;
|
||||
fs::create_dir_all(dest).map_err(|e| format!("create rootfs dir {}: {e}", dest.display()))?;
|
||||
let file =
|
||||
File::open(archive_path).map_err(|e| format!("open {}: {e}", archive_path.display()))?;
|
||||
let mut archive = tar::Archive::new(file);
|
||||
archive
|
||||
.unpack(dest)
|
||||
.map_err(|e| format!("extract rootfs tarball into {}: {e}", dest.display()))
|
||||
}
|
||||
|
||||
pub fn create_rootfs_archive_from_dir(source: &Path, archive_path: &Path) -> Result<(), String> {
|
||||
if let Some(parent) = archive_path.parent() {
|
||||
fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?;
|
||||
}
|
||||
|
||||
let file = File::create(archive_path)
|
||||
.map_err(|e| format!("create {}: {e}", archive_path.display()))?;
|
||||
let writer = BufWriter::new(file);
|
||||
let mut builder = tar::Builder::new(writer);
|
||||
append_rootfs_tree_to_archive(&mut builder, source, Path::new("")).map_err(|e| {
|
||||
format!(
|
||||
"archive {} into {}: {e}",
|
||||
source.display(),
|
||||
archive_path.display()
|
||||
)
|
||||
})?;
|
||||
builder
|
||||
.finish()
|
||||
.map_err(|e| format!("finalize {}: {e}", archive_path.display()))
|
||||
}
|
||||
|
||||
fn append_rootfs_tree_to_archive(
|
||||
builder: &mut tar::Builder<BufWriter<File>>,
|
||||
source: &Path,
|
||||
archive_prefix: &Path,
|
||||
) -> Result<(), String> {
|
||||
let mut entries = fs::read_dir(source)
|
||||
.map_err(|e| format!("read {}: {e}", source.display()))?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| format!("read {}: {e}", source.display()))?;
|
||||
entries.sort_by_key(fs::DirEntry::file_name);
|
||||
|
||||
for entry in entries {
|
||||
let entry_name = entry.file_name();
|
||||
let source_path = entry.path();
|
||||
let archive_path = if archive_prefix.as_os_str().is_empty() {
|
||||
entry_name.into()
|
||||
} else {
|
||||
archive_prefix.join(entry_name)
|
||||
};
|
||||
let metadata = fs::symlink_metadata(&source_path)
|
||||
.map_err(|e| format!("stat {}: {e}", source_path.display()))?;
|
||||
let file_type = metadata.file_type();
|
||||
|
||||
if file_type.is_dir() {
|
||||
builder
|
||||
.append_dir(&archive_path, &source_path)
|
||||
.map_err(|e| format!("append dir {}: {e}", source_path.display()))?;
|
||||
append_rootfs_tree_to_archive(builder, &source_path, &archive_path)?;
|
||||
continue;
|
||||
}
|
||||
|
||||
if file_type.is_file() {
|
||||
let mut file = File::open(&source_path)
|
||||
.map_err(|e| format!("open {}: {e}", source_path.display()))?;
|
||||
builder
|
||||
.append_file(&archive_path, &mut file)
|
||||
.map_err(|e| format!("append file {}: {e}", source_path.display()))?;
|
||||
continue;
|
||||
}
|
||||
|
||||
if file_type.is_symlink() {
|
||||
append_symlink_to_archive(builder, &source_path, &archive_path, &metadata)?;
|
||||
continue;
|
||||
}
|
||||
|
||||
return Err(format!(
|
||||
"unsupported rootfs entry type at {}",
|
||||
source_path.display()
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn unpack_zstd_tar(blob: &[u8], label: &str, dest: &Path) -> Result<(), String> {
|
||||
fs::create_dir_all(dest).map_err(|e| format!("create rootfs dir {}: {e}", dest.display()))?;
|
||||
|
||||
let decoder = zstd::Decoder::new(Cursor::new(blob))
|
||||
.map_err(|e| format!("decompress {label} rootfs: {e}"))?;
|
||||
let mut archive = tar::Archive::new(decoder);
|
||||
archive.unpack(dest).map_err(|e| {
|
||||
format!(
|
||||
"extract {label} rootfs tarball into {}: {e}",
|
||||
dest.display()
|
||||
)
|
||||
})
|
||||
fn append_symlink_to_archive(
|
||||
builder: &mut tar::Builder<BufWriter<File>>,
|
||||
source_path: &Path,
|
||||
archive_path: &Path,
|
||||
metadata: &fs::Metadata,
|
||||
) -> Result<(), String> {
|
||||
let target = fs::read_link(source_path)
|
||||
.map_err(|e| format!("readlink {}: {e}", source_path.display()))?;
|
||||
let mut header = tar::Header::new_gnu();
|
||||
header.set_metadata(metadata);
|
||||
header.set_size(0);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_link(&mut header, archive_path, target)
|
||||
.map_err(|e| format!("append symlink {}: {e}", source_path.display()))
|
||||
}
|
||||
|
||||
fn prepare_sandbox_rootfs(rootfs: &Path) -> Result<(), String> {
|
||||
@@ -103,17 +167,30 @@ fn prepare_sandbox_rootfs(rootfs: &Path) -> Result<(), String> {
|
||||
.map_err(|e| format!("chmod {}: {e}", init_path.display()))?;
|
||||
}
|
||||
|
||||
ensure_supervisor_binary(rootfs)?;
|
||||
|
||||
let opt_dir = rootfs.join("opt/openshell");
|
||||
fs::create_dir_all(&opt_dir).map_err(|e| format!("create {}: {e}", opt_dir.display()))?;
|
||||
fs::write(opt_dir.join(".rootfs-type"), "sandbox\n")
|
||||
.map_err(|e| format!("write sandbox rootfs marker: {e}"))?;
|
||||
ensure_sandbox_guest_user(rootfs)?;
|
||||
fs::create_dir_all(rootfs.join("sandbox"))
|
||||
.map_err(|e| format!("create sandbox workdir: {e}"))?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_sandbox_rootfs(rootfs: &Path) -> Result<(), String> {
|
||||
require_rootfs_path(rootfs, SANDBOX_GUEST_INIT_PATH)?;
|
||||
require_rootfs_path(rootfs, "/opt/openshell/bin/openshell-sandbox")?;
|
||||
require_any_rootfs_path(rootfs, &["/bin/bash"])?;
|
||||
require_any_rootfs_path(rootfs, &["/bin/mount", "/usr/bin/mount"])?;
|
||||
require_any_rootfs_path(
|
||||
rootfs,
|
||||
&["/sbin/ip", "/usr/sbin/ip", "/bin/ip", "/usr/bin/ip"],
|
||||
)?;
|
||||
require_any_rootfs_path(rootfs, &["/bin/sed", "/usr/bin/sed"])?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn ensure_sandbox_guest_user(rootfs: &Path) -> Result<(), String> {
|
||||
const SANDBOX_UID: u32 = 10001;
|
||||
const SANDBOX_GID: u32 = 10001;
|
||||
@@ -167,6 +244,62 @@ fn ensure_line_in_file(
|
||||
fs::write(path, contents).map_err(|e| format!("write {}: {e}", path.display()))
|
||||
}
|
||||
|
||||
fn ensure_supervisor_binary(rootfs: &Path) -> Result<(), String> {
|
||||
let path = rootfs.join(SANDBOX_SUPERVISOR_PATH.trim_start_matches('/'));
|
||||
if SUPERVISOR.is_empty() {
|
||||
if !path.exists() {
|
||||
return Err(
|
||||
"sandbox supervisor not embedded. Build openshell-driver-vm with OPENSHELL_VM_RUNTIME_COMPRESSED_DIR set and run `mise run vm:setup && mise run vm:supervisor` first"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
} else {
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent).map_err(|e| format!("create {}: {e}", parent.display()))?;
|
||||
}
|
||||
|
||||
let supervisor = zstd::decode_all(Cursor::new(SUPERVISOR))
|
||||
.map_err(|e| format!("decompress supervisor: {e}"))?;
|
||||
fs::write(&path, supervisor).map_err(|e| format!("write {}: {e}", path.display()))?;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt as _;
|
||||
|
||||
fs::set_permissions(&path, fs::Permissions::from_mode(0o755))
|
||||
.map_err(|e| format!("chmod {}: {e}", path.display()))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn require_rootfs_path(rootfs: &Path, relative: &str) -> Result<(), String> {
|
||||
let candidate = rootfs.join(relative.trim_start_matches('/'));
|
||||
if candidate.exists() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"prepared rootfs is missing {}",
|
||||
candidate.display()
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn require_any_rootfs_path(rootfs: &Path, candidates: &[&str]) -> Result<(), String> {
|
||||
if candidates
|
||||
.iter()
|
||||
.any(|candidate| rootfs.join(candidate.trim_start_matches('/')).exists())
|
||||
{
|
||||
Ok(())
|
||||
} else {
|
||||
Err(format!(
|
||||
"prepared rootfs is missing one of: {}",
|
||||
candidates.join(", ")
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_rootfs_path(rootfs: &Path, relative: &str) -> Result<(), String> {
|
||||
let path = rootfs.join(relative);
|
||||
if !path.exists() {
|
||||
@@ -198,9 +331,15 @@ mod tests {
|
||||
fs::create_dir_all(rootfs.join("var/lib/rancher")).expect("create var/lib/rancher");
|
||||
fs::create_dir_all(rootfs.join("opt/openshell/charts")).expect("create charts");
|
||||
fs::create_dir_all(rootfs.join("opt/openshell/manifests")).expect("create manifests");
|
||||
fs::create_dir_all(rootfs.join("opt/openshell/bin")).expect("create openshell bin");
|
||||
fs::write(rootfs.join("usr/local/bin/k3s"), b"k3s").expect("write k3s");
|
||||
fs::write(rootfs.join("usr/local/bin/kubectl"), b"kubectl").expect("write kubectl");
|
||||
fs::write(rootfs.join("opt/openshell/.initialized"), b"yes").expect("write initialized");
|
||||
fs::write(
|
||||
rootfs.join("opt/openshell/bin/openshell-sandbox"),
|
||||
b"sandbox",
|
||||
)
|
||||
.expect("write openshell-sandbox");
|
||||
fs::write(
|
||||
rootfs.join("etc/passwd"),
|
||||
"root:x:0:0:root:/root:/bin/bash\n",
|
||||
@@ -208,8 +347,15 @@ mod tests {
|
||||
.expect("write passwd");
|
||||
fs::write(rootfs.join("etc/group"), "root:x:0:\n").expect("write group");
|
||||
fs::write(rootfs.join("etc/hosts"), "127.0.0.1 localhost\n").expect("write hosts");
|
||||
fs::create_dir_all(rootfs.join("bin")).expect("create bin");
|
||||
fs::create_dir_all(rootfs.join("sbin")).expect("create sbin");
|
||||
fs::write(rootfs.join("bin/bash"), b"bash").expect("write bash");
|
||||
fs::write(rootfs.join("bin/mount"), b"mount").expect("write mount");
|
||||
fs::write(rootfs.join("bin/sed"), b"sed").expect("write sed");
|
||||
fs::write(rootfs.join("sbin/ip"), b"ip").expect("write ip");
|
||||
|
||||
prepare_sandbox_rootfs(&rootfs).expect("prepare sandbox rootfs");
|
||||
validate_sandbox_rootfs(&rootfs).expect("validate sandbox rootfs");
|
||||
|
||||
assert!(!rootfs.join("usr/local/bin/k3s").exists());
|
||||
assert!(!rootfs.join("usr/local/bin/kubectl").exists());
|
||||
@@ -217,7 +363,7 @@ mod tests {
|
||||
assert!(!rootfs.join("opt/openshell/charts").exists());
|
||||
assert!(!rootfs.join("opt/openshell/manifests").exists());
|
||||
assert!(rootfs.join("srv/openshell-vm-sandbox-init.sh").is_file());
|
||||
assert!(rootfs.join("sandbox").is_dir());
|
||||
assert!(!rootfs.join("sandbox").exists());
|
||||
assert!(
|
||||
fs::read_to_string(rootfs.join("etc/passwd"))
|
||||
.expect("read passwd")
|
||||
@@ -236,6 +382,61 @@ mod tests {
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_sandbox_rootfs_preserves_image_workdir_contents() {
|
||||
let dir = unique_temp_dir();
|
||||
let rootfs = dir.join("rootfs");
|
||||
|
||||
fs::create_dir_all(rootfs.join("opt/openshell/bin")).expect("create openshell bin");
|
||||
fs::write(
|
||||
rootfs.join("opt/openshell/bin/openshell-sandbox"),
|
||||
b"sandbox",
|
||||
)
|
||||
.expect("write openshell-sandbox");
|
||||
fs::create_dir_all(rootfs.join("sandbox")).expect("create sandbox workdir");
|
||||
fs::write(rootfs.join("sandbox/app.py"), "print('hello')\n").expect("write app");
|
||||
|
||||
prepare_sandbox_rootfs(&rootfs).expect("prepare sandbox rootfs");
|
||||
|
||||
assert_eq!(
|
||||
fs::read_to_string(rootfs.join("sandbox/app.py")).expect("read app"),
|
||||
"print('hello')\n"
|
||||
);
|
||||
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn create_rootfs_archive_preserves_broken_symlinks() {
|
||||
let dir = unique_temp_dir();
|
||||
let rootfs = dir.join("rootfs");
|
||||
let extracted = dir.join("extracted");
|
||||
let archive = dir.join("rootfs.tar");
|
||||
|
||||
fs::create_dir_all(rootfs.join("etc")).expect("create etc");
|
||||
fs::write(rootfs.join("etc/hosts"), "127.0.0.1 localhost\n").expect("write hosts");
|
||||
std::os::unix::fs::symlink("/proc/self/mounts", rootfs.join("etc/mtab"))
|
||||
.expect("create symlink");
|
||||
|
||||
create_rootfs_archive_from_dir(&rootfs, &archive).expect("archive rootfs");
|
||||
extract_rootfs_archive_to(&archive, &extracted).expect("extract rootfs");
|
||||
|
||||
let extracted_link = extracted.join("etc/mtab");
|
||||
assert!(
|
||||
fs::symlink_metadata(&extracted_link)
|
||||
.unwrap()
|
||||
.file_type()
|
||||
.is_symlink()
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read_link(&extracted_link).expect("read extracted symlink"),
|
||||
PathBuf::from("/proc/self/mounts")
|
||||
);
|
||||
|
||||
let _ = fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
fn unique_temp_dir() -> PathBuf {
|
||||
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
let nanos = SystemTime::now()
|
||||
|
||||
@@ -110,7 +110,8 @@ fn run_qemu_vm(config: &VmLaunchConfig) -> Result<(), String> {
|
||||
#[cfg(target_os = "linux")]
|
||||
check_kvm_access()?;
|
||||
|
||||
write_guest_env_file(&config.rootfs, &config.env)?;
|
||||
let guest_env = qemu_guest_env_vars(config, host_dns_server());
|
||||
write_guest_env_file(&config.rootfs, &guest_env)?;
|
||||
|
||||
let rootfs_str = config.rootfs.to_str().ok_or("rootfs path not UTF-8")?;
|
||||
let sandbox_dir = config.rootfs.parent().unwrap_or(&config.rootfs);
|
||||
@@ -296,6 +297,27 @@ fn write_guest_env_file(rootfs: &Path, env_vars: &[String]) -> Result<(), String
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn qemu_guest_env_vars(config: &VmLaunchConfig, dns_server: Option<String>) -> Vec<String> {
|
||||
let mut env_vars = config.env.clone();
|
||||
|
||||
if let Some(ip) = &config.guest_ip
|
||||
&& let Some(host_ip) = &config.host_ip
|
||||
{
|
||||
env_vars.push(format!("VM_NET_IP={ip}"));
|
||||
env_vars.push(format!("VM_NET_GW={host_ip}"));
|
||||
}
|
||||
|
||||
if let Some(dns) = dns_server {
|
||||
env_vars.push(format!("VM_NET_DNS={dns}"));
|
||||
}
|
||||
|
||||
if config.gpu_bdf.is_some() {
|
||||
env_vars.push("GPU_ENABLED=true".to_string());
|
||||
}
|
||||
|
||||
env_vars
|
||||
}
|
||||
|
||||
/// Escape a string for use inside bash double quotes.
|
||||
fn shell_escape(s: &str) -> String {
|
||||
s.replace('\\', "\\\\")
|
||||
@@ -320,16 +342,9 @@ fn build_kernel_cmdline(config: &VmLaunchConfig) -> String {
|
||||
&& let Some(host_ip) = &config.host_ip
|
||||
{
|
||||
parts.push(format!("ip={ip}::{host_ip}:255.255.255.252:sandbox::off"));
|
||||
parts.push(format!("VM_NET_IP={ip}"));
|
||||
parts.push(format!("VM_NET_GW={host_ip}"));
|
||||
}
|
||||
|
||||
if let Some(dns) = host_dns_server() {
|
||||
parts.push(format!("VM_NET_DNS={dns}"));
|
||||
}
|
||||
|
||||
if config.gpu_bdf.is_some() {
|
||||
parts.push("GPU_ENABLED=true".to_string());
|
||||
parts.push("firmware_class.path=/lib/firmware".to_string());
|
||||
}
|
||||
|
||||
@@ -705,13 +720,15 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> {
|
||||
// talks to on boot (IPs 192.168.127.1 / .2, defaults for
|
||||
// gvisor-tap-vsock);
|
||||
// * the host-facing gateway identity the guest uses for callbacks:
|
||||
// the init script seeds `/etc/hosts` with
|
||||
// `host.openshell.internal` pointing at 192.168.127.1 while
|
||||
// leaving gvproxy's legacy `host.containers.internal` /
|
||||
// `host.docker.internal` DNS answers intact, which is how the guest's
|
||||
// `rewrite_openshell_endpoint_if_needed` probe reaches the host
|
||||
// gateway when the bare loopback address doesn't resolve from
|
||||
// inside the VM.
|
||||
// gvproxy installs a default NAT entry rewriting `192.168.127.254`
|
||||
// (the subnet's HostIP) to the host's `127.0.0.1`, and serves
|
||||
// `host.containers.internal` / `host.docker.internal` /
|
||||
// `host.openshell.internal` in its embedded DNS pointing at that
|
||||
// same HostIP. The guest init script seeds /etc/hosts with the
|
||||
// same mapping so the supervisor reaches the host gateway even
|
||||
// when gvproxy's DNS isn't in resolv.conf. The gateway IP
|
||||
// (192.168.127.1) is NOT a host-loopback proxy — it only listens
|
||||
// on its own service ports (DNS:53, DHCP, HTTP API:80).
|
||||
//
|
||||
// That network plane is also what the sandbox supervisor's
|
||||
// per-sandbox netns (veth pair + iptables, see
|
||||
@@ -1318,3 +1335,53 @@ fn check_kvm_access() -> Result<(), String> {
|
||||
format!("cannot open /dev/kvm: {e}\nKVM access is required to run microVMs on Linux.")
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn qemu_config() -> VmLaunchConfig {
|
||||
VmLaunchConfig {
|
||||
rootfs: PathBuf::from("/rootfs"),
|
||||
vcpus: 2,
|
||||
mem_mib: 2048,
|
||||
exec_path: "/srv/openshell-vm-sandbox-init.sh".to_string(),
|
||||
args: Vec::new(),
|
||||
env: vec!["OPENSHELL_ENDPOINT=http://10.0.128.1:8080".to_string()],
|
||||
workdir: "/".to_string(),
|
||||
log_level: 0,
|
||||
console_output: PathBuf::from("/console.log"),
|
||||
backend: VmBackend::Qemu,
|
||||
gpu_bdf: Some("0000:01:00.0".to_string()),
|
||||
tap_device: Some("vmtap-test".to_string()),
|
||||
guest_ip: Some("10.0.128.2".to_string()),
|
||||
host_ip: Some("10.0.128.1".to_string()),
|
||||
vsock_cid: Some(4),
|
||||
guest_mac: Some("02:00:00:00:00:01".to_string()),
|
||||
gateway_port: Some(8080),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn qemu_guest_env_vars_include_driver_runtime_metadata() {
|
||||
let env = qemu_guest_env_vars(&qemu_config(), Some("1.1.1.1".to_string()));
|
||||
|
||||
assert!(env.contains(&"OPENSHELL_ENDPOINT=http://10.0.128.1:8080".to_string()));
|
||||
assert!(env.contains(&"VM_NET_IP=10.0.128.2".to_string()));
|
||||
assert!(env.contains(&"VM_NET_GW=10.0.128.1".to_string()));
|
||||
assert!(env.contains(&"VM_NET_DNS=1.1.1.1".to_string()));
|
||||
assert!(env.contains(&"GPU_ENABLED=true".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kernel_cmdline_keeps_guest_init_metadata_out_of_proc_cmdline() {
|
||||
let cmdline = build_kernel_cmdline(&qemu_config());
|
||||
|
||||
assert!(cmdline.contains("ip=10.0.128.2::10.0.128.1:255.255.255.252:sandbox::off"));
|
||||
assert!(cmdline.contains("firmware_class.path=/lib/firmware"));
|
||||
assert!(!cmdline.contains("VM_NET_IP="));
|
||||
assert!(!cmdline.contains("VM_NET_GW="));
|
||||
assert!(!cmdline.contains("VM_NET_DNS="));
|
||||
assert!(!cmdline.contains("GPU_ENABLED="));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Under sudo, PATH is reset and user-local tools (mise, cargo) disappear.
|
||||
# Restore the invoking user's tool directories so mise and its shims work.
|
||||
if [ -n "${SUDO_USER:-}" ]; then
|
||||
_sudo_home=$(getent passwd "${SUDO_USER}" | cut -d: -f6)
|
||||
for _p in "${_sudo_home}/.local/bin" "${_sudo_home}/.local/share/mise/shims" "${_sudo_home}/.cargo/bin"; do
|
||||
[ -d "${_p}" ] && PATH="${_p}:${PATH}"
|
||||
done
|
||||
export PATH
|
||||
fi
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
CLI_BIN="${ROOT}/scripts/bin/openshell"
|
||||
COMPRESSED_DIR="${ROOT}/target/vm-runtime-compressed"
|
||||
SERVER_PORT="${OPENSHELL_SERVER_PORT:-8080}"
|
||||
# Keep the driver socket path under AF_UNIX SUN_LEN on macOS.
|
||||
STATE_DIR_ROOT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}"
|
||||
STATE_LABEL_RAW="${OPENSHELL_VM_INSTANCE:-port-${SERVER_PORT}}"
|
||||
STATE_LABEL="$(printf '%s' "${STATE_LABEL_RAW}" | tr -cs '[:alnum:]._-' '-')"
|
||||
if [ -z "${STATE_LABEL}" ]; then
|
||||
STATE_LABEL="port-${SERVER_PORT}"
|
||||
fi
|
||||
STATE_DIR_DEFAULT="${STATE_DIR_ROOT}/openshell-vm-driver-dev-${USER:-user}-${STATE_LABEL}"
|
||||
STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${STATE_DIR_DEFAULT}}"
|
||||
DB_PATH_DEFAULT="${STATE_DIR}/openshell.db"
|
||||
VM_HOST_GATEWAY_DEFAULT="${OPENSHELL_VM_HOST_GATEWAY:-host.containers.internal}"
|
||||
LOCAL_GATEWAY_ENDPOINT_DEFAULT="http://127.0.0.1:${SERVER_PORT}"
|
||||
LOCAL_GATEWAY_ENDPOINT="${OPENSHELL_VM_LOCAL_GATEWAY_ENDPOINT:-${LOCAL_GATEWAY_ENDPOINT_DEFAULT}}"
|
||||
GATEWAY_NAME_DEFAULT="vm-driver-${STATE_LABEL}"
|
||||
GATEWAY_NAME="${OPENSHELL_VM_GATEWAY_NAME:-${GATEWAY_NAME_DEFAULT}}"
|
||||
DRIVER_DIR_DEFAULT="${ROOT}/target/debug"
|
||||
DRIVER_DIR="${OPENSHELL_DRIVER_DIR:-${DRIVER_DIR_DEFAULT}}"
|
||||
|
||||
export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR}}"
|
||||
|
||||
for arg in "$@"; do
|
||||
if [ "${arg}" = "--gpu" ]; then
|
||||
export OPENSHELL_VM_GPU=true
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
mkdir -p "${STATE_DIR}"
|
||||
|
||||
normalize_bool() {
|
||||
case "${1,,}" in
|
||||
1|true|yes|on) echo "true" ;;
|
||||
0|false|no|off) echo "false" ;;
|
||||
*)
|
||||
echo "invalid boolean value '$1' (expected true/false, 1/0, yes/no, on/off)" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
check_supervisor_cross_toolchain() {
|
||||
# The sandbox supervisor inside the guest is always Linux. On non-Linux
|
||||
# hosts (macOS) and on Linux hosts with a different arch than the guest,
|
||||
# we cross-compile via cargo-zigbuild and need the matching rustup target.
|
||||
local host_os host_arch guest_arch rust_target
|
||||
host_os="$(uname -s)"
|
||||
host_arch="$(uname -m)"
|
||||
guest_arch="${GUEST_ARCH:-${host_arch}}"
|
||||
case "${guest_arch}" in
|
||||
arm64|aarch64) rust_target="aarch64-unknown-linux-gnu" ;;
|
||||
x86_64|amd64) rust_target="x86_64-unknown-linux-gnu" ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
if [ "${host_os}" = "Linux" ] && [ "${host_arch}" = "${guest_arch}" ]; then
|
||||
return 0
|
||||
fi
|
||||
local missing=0
|
||||
if ! command -v cargo-zigbuild >/dev/null 2>&1; then
|
||||
echo "ERROR: cargo-zigbuild not found (required to cross-compile the guest supervisor)." >&2
|
||||
echo " Install: cargo install --locked cargo-zigbuild && brew install zig" >&2
|
||||
missing=1
|
||||
fi
|
||||
if ! rustup target list --installed 2>/dev/null | grep -qx "${rust_target}"; then
|
||||
echo "ERROR: Rust target '${rust_target}' not installed." >&2
|
||||
echo " Install: rustup target add ${rust_target}" >&2
|
||||
missing=1
|
||||
fi
|
||||
if [ "${missing}" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
if [ ! -s "${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR}/rootfs.tar.zst" ]; then
|
||||
check_supervisor_cross_toolchain
|
||||
echo "==> Building base VM rootfs tarball"
|
||||
mise run vm:rootfs -- --base
|
||||
fi
|
||||
|
||||
if [ "${OPENSHELL_VM_GPU:-}" = "true" ] && [ ! -s "${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR}/rootfs-gpu.tar.zst" ]; then
|
||||
check_supervisor_cross_toolchain
|
||||
echo "==> Building GPU VM rootfs tarball"
|
||||
mise run vm:rootfs -- --gpu
|
||||
fi
|
||||
|
||||
if [ ! -s "${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR}/rootfs.tar.zst" ] || ! find "${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q .; then
|
||||
echo "==> Preparing embedded VM runtime"
|
||||
mise run vm:setup
|
||||
fi
|
||||
|
||||
echo "==> Building gateway and VM compute driver"
|
||||
cargo build -p openshell-server -p openshell-driver-vm
|
||||
|
||||
if [ "$(uname -s)" = "Darwin" ]; then
|
||||
echo "==> Codesigning VM compute driver"
|
||||
codesign \
|
||||
--entitlements "${ROOT}/crates/openshell-driver-vm/entitlements.plist" \
|
||||
--force \
|
||||
-s - \
|
||||
"${ROOT}/target/debug/openshell-driver-vm"
|
||||
fi
|
||||
|
||||
export OPENSHELL_DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")"
|
||||
export OPENSHELL_DB_URL="${OPENSHELL_DB_URL:-sqlite:${DB_PATH_DEFAULT}}"
|
||||
export OPENSHELL_DRIVERS="${OPENSHELL_DRIVERS:-vm}"
|
||||
export OPENSHELL_DRIVER_DIR="${DRIVER_DIR}"
|
||||
export OPENSHELL_GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-http://${VM_HOST_GATEWAY_DEFAULT}:${SERVER_PORT}}"
|
||||
export OPENSHELL_SSH_GATEWAY_HOST="${OPENSHELL_SSH_GATEWAY_HOST:-127.0.0.1}"
|
||||
export OPENSHELL_SSH_GATEWAY_PORT="${OPENSHELL_SSH_GATEWAY_PORT:-${SERVER_PORT}}"
|
||||
export OPENSHELL_SSH_HANDSHAKE_SECRET="${OPENSHELL_SSH_HANDSHAKE_SECRET:-}"
|
||||
export OPENSHELL_VM_DRIVER_STATE_DIR="${STATE_DIR}"
|
||||
|
||||
# Resolve the VM runtime directory (contains vmlinux, virtiofsd, etc.)
|
||||
# so the child --internal-run-vm process can find it under sudo.
|
||||
if [ -z "${OPENSHELL_VM_RUNTIME_DIR:-}" ]; then
|
||||
_candidate="${HOME}/.local/share/openshell/vm-runtime/0.0.0"
|
||||
if [ -n "${SUDO_USER:-}" ]; then
|
||||
_sudo_home=$(getent passwd "${SUDO_USER}" | cut -d: -f6)
|
||||
_candidate="${_sudo_home}/.local/share/openshell/vm-runtime/0.0.0"
|
||||
fi
|
||||
if [ -f "${_candidate}/vmlinux" ]; then
|
||||
export OPENSHELL_VM_RUNTIME_DIR="${_candidate}"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "==> Registering gateway"
|
||||
echo " Name: ${GATEWAY_NAME}"
|
||||
echo " Endpoint: ${LOCAL_GATEWAY_ENDPOINT}"
|
||||
echo " Driver: ${OPENSHELL_DRIVER_DIR}/openshell-driver-vm"
|
||||
|
||||
# GPU passthrough requires root, but gateway config must be written to the
|
||||
# real user's home directory — not /root/.config/openshell/.
|
||||
# Unset XDG_CONFIG_HOME so the CLI falls back to $HOME/.config (sudo -u
|
||||
# sets HOME correctly but may inherit XDG_CONFIG_HOME from the root env).
|
||||
if [ -n "${SUDO_USER:-}" ]; then
|
||||
sudo -u "${SUDO_USER}" env -u XDG_CONFIG_HOME "PATH=${PATH}" "${CLI_BIN}" gateway destroy --name "${GATEWAY_NAME}" 2>/dev/null || true
|
||||
sudo -u "${SUDO_USER}" env -u XDG_CONFIG_HOME "PATH=${PATH}" "${CLI_BIN}" gateway add --name "${GATEWAY_NAME}" "${LOCAL_GATEWAY_ENDPOINT}"
|
||||
sudo -u "${SUDO_USER}" env -u XDG_CONFIG_HOME "PATH=${PATH}" "${CLI_BIN}" gateway select "${GATEWAY_NAME}"
|
||||
else
|
||||
"${CLI_BIN}" gateway destroy --name "${GATEWAY_NAME}" 2>/dev/null || true
|
||||
"${CLI_BIN}" gateway add --name "${GATEWAY_NAME}" "${LOCAL_GATEWAY_ENDPOINT}"
|
||||
"${CLI_BIN}" gateway select "${GATEWAY_NAME}"
|
||||
fi
|
||||
|
||||
echo "==> Starting OpenShell server with VM compute driver"
|
||||
exec "${ROOT}/target/debug/openshell-gateway"
|
||||
@@ -407,6 +407,7 @@ async fn run_from_args(args: Args) -> Result<()> {
|
||||
let vm_config = VmComputeConfig {
|
||||
state_dir: args.vm_driver_state_dir,
|
||||
driver_dir: args.driver_dir,
|
||||
default_image: config.sandbox_image.clone(),
|
||||
krun_log_level: args.vm_krun_log_level,
|
||||
vcpus: args.vm_vcpus,
|
||||
mem_mib: args.vm_mem_mib,
|
||||
|
||||
@@ -63,6 +63,9 @@ pub struct VmComputeConfig {
|
||||
/// falls back to its conventional install paths and sibling binary.
|
||||
pub driver_dir: Option<PathBuf>,
|
||||
|
||||
/// Default sandbox image the driver should use when a request omits one.
|
||||
pub default_image: String,
|
||||
|
||||
/// libkrun log level used by the VM driver helper.
|
||||
pub krun_log_level: u32,
|
||||
|
||||
@@ -125,6 +128,7 @@ impl Default for VmComputeConfig {
|
||||
Self {
|
||||
state_dir: Self::default_state_dir(),
|
||||
driver_dir: None,
|
||||
default_image: String::new(),
|
||||
krun_log_level: Self::default_krun_log_level(),
|
||||
vcpus: Self::default_vcpus(),
|
||||
mem_mib: Self::default_mem_mib(),
|
||||
@@ -304,9 +308,17 @@ pub async fn spawn(
|
||||
.arg("--openshell-endpoint")
|
||||
.arg(&config.grpc_endpoint);
|
||||
command.arg("--state-dir").arg(&vm_config.state_dir);
|
||||
command
|
||||
.arg("--ssh-handshake-secret")
|
||||
.arg(&config.ssh_handshake_secret);
|
||||
if !vm_config.default_image.trim().is_empty() {
|
||||
command.arg("--default-image").arg(&vm_config.default_image);
|
||||
}
|
||||
// Only forward the handshake secret when one is configured. The VM
|
||||
// driver does not consume it, but accepts it for parity with the
|
||||
// Kubernetes/Podman drivers; passing an empty value is noise.
|
||||
if !config.ssh_handshake_secret.is_empty() {
|
||||
command
|
||||
.arg("--ssh-handshake-secret")
|
||||
.arg(&config.ssh_handshake_secret);
|
||||
}
|
||||
command
|
||||
.arg("--ssh-handshake-skew-secs")
|
||||
.arg(config.ssh_handshake_skew_secs.to_string());
|
||||
|
||||
@@ -156,6 +156,15 @@ pub async fn run_server(
|
||||
if database_url.is_empty() {
|
||||
return Err(Error::config("database_url is required"));
|
||||
}
|
||||
let driver = configured_compute_driver(&config)?;
|
||||
if config.ssh_handshake_secret.is_empty()
|
||||
&& !matches!(driver, ComputeDriverKind::Docker | ComputeDriverKind::Vm)
|
||||
{
|
||||
return Err(Error::config(
|
||||
"ssh_handshake_secret is required. Set --ssh-handshake-secret or OPENSHELL_SSH_HANDSHAKE_SECRET",
|
||||
));
|
||||
}
|
||||
|
||||
let store = Arc::new(Store::connect(database_url).await?);
|
||||
|
||||
let oidc_cache = if let Some(ref oidc) = config.oidc {
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
#
|
||||
# openshell-driver-vm loads libkrun/libkrunfw at runtime via dlopen, so it
|
||||
# does NOT need Hypervisor.framework headers at build time. Pre-compressed
|
||||
# runtime artifacts (libkrun, libkrunfw, gvproxy, rootfs) are injected via
|
||||
# runtime artifacts (libkrun, libkrunfw, gvproxy, bundled supervisor) are injected via
|
||||
# the vm-runtime-compressed build context and embedded into the binary via
|
||||
# include_bytes!().
|
||||
#
|
||||
|
||||
@@ -58,8 +58,6 @@ The `--from` flag also accepts:
|
||||
openshell sandbox create --from ./my-sandbox-dir
|
||||
```
|
||||
|
||||
Local Dockerfile sources build into the host Docker daemon. Use them with a Docker-backed gateway, or publish the image to a registry and pass the registry reference.
|
||||
|
||||
- Container image references: Use an existing container image directly:
|
||||
|
||||
```shell
|
||||
|
||||
@@ -55,7 +55,9 @@ openshell sandbox create --from my-registry.example.com/my-image:latest
|
||||
|
||||
The CLI resolves community names against the [OpenShell Community](https://github.com/NVIDIA/OpenShell-Community) catalog, pulls the bundled Dockerfile and policy, builds the image locally, and creates the sandbox. For the full catalog and how to contribute your own, refer to [Community Sandboxes](/sandboxes/community-sandboxes).
|
||||
|
||||
Local Dockerfile sources build into the host Docker daemon. Use them with a Docker-backed gateway, or publish the image to a registry and pass the registry reference.
|
||||
Local directories and Dockerfiles require a local gateway because the CLI builds
|
||||
through the local Docker daemon. Use a registry image reference for remote
|
||||
gateways.
|
||||
|
||||
### Label a Sandbox
|
||||
|
||||
|
||||
+8
-1
@@ -167,13 +167,20 @@ echo "==> Starting openshell-gateway on 127.0.0.1:${HOST_PORT} (state: ${RUN_STA
|
||||
# `~/.local/libexec/openshell/openshell-driver-vm` when present
|
||||
# (install-vm.sh installs there), which silently shadows development
|
||||
# builds — a subtle source of stale-binary bugs in e2e runs.
|
||||
# --grpc-endpoint is the URL the VM driver passes into each guest as
|
||||
# OPENSHELL_ENDPOINT. The supervisor inside the VM dials this address.
|
||||
# Use `host.containers.internal` rather than `127.0.0.1` so gvproxy's
|
||||
# host-loopback proxy carries the connection — gvproxy's bare gateway IP
|
||||
# (192.168.127.1) does NOT forward arbitrary host ports. The driver also
|
||||
# rewrites loopback URLs to this hostname as a safety net, so this matches
|
||||
# what the guest will actually see and aligns with `tasks/scripts/gateway-vm.sh`.
|
||||
"${GATEWAY_BIN}" \
|
||||
--drivers vm \
|
||||
--disable-tls \
|
||||
--disable-gateway-auth \
|
||||
--db-url 'sqlite::memory:' \
|
||||
--port "${HOST_PORT}" \
|
||||
--grpc-endpoint "http://127.0.0.1:${HOST_PORT}" \
|
||||
--grpc-endpoint "http://host.containers.internal:${HOST_PORT}" \
|
||||
--ssh-handshake-secret "${SSH_HANDSHAKE_SECRET}" \
|
||||
--driver-dir "${ROOT}/target/debug" \
|
||||
--vm-driver-state-dir "${RUN_STATE_DIR}" \
|
||||
|
||||
@@ -20,8 +20,9 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
RUN groupadd -g 1000660000 sandbox && \
|
||||
useradd -m -u 1000660000 -g sandbox sandbox
|
||||
|
||||
RUN install -d -o sandbox -g sandbox /sandbox
|
||||
WORKDIR /sandbox
|
||||
COPY app.py .
|
||||
COPY --chown=sandbox:sandbox app.py .
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
|
||||
@@ -62,6 +62,8 @@ key requirements are:
|
||||
- **Create a `sandbox` user** (uid/gid 1000660000) for non-root execution.
|
||||
Use a high UID (1000000000+) to avoid conflicts with host users when running
|
||||
without user namespace remapping.
|
||||
- **Make your application workdir writable by `sandbox`**. This example creates
|
||||
`/sandbox` with `sandbox:sandbox` ownership before copying `app.py`.
|
||||
- **Install `iproute2`** for full network namespace isolation.
|
||||
- **Use a standard Linux base image** — distroless and `FROM scratch`
|
||||
images are not supported.
|
||||
|
||||
+10
-2
@@ -5,6 +5,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
|
||||
BINARY="$PROJECT_ROOT/target/debug/openshell"
|
||||
STATE_FILE="$PROJECT_ROOT/.cache/openshell-build.state"
|
||||
CALLER_PWD="$PWD"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fingerprint-based rebuild check
|
||||
@@ -26,7 +27,10 @@ else
|
||||
current_head=$(git rev-parse HEAD 2>/dev/null || echo "unknown")
|
||||
|
||||
# Collect dirty (modified, staged, untracked) files
|
||||
mapfile -t changed_files < <(
|
||||
changed_files=()
|
||||
while IFS= read -r path; do
|
||||
changed_files+=("$path")
|
||||
done < <(
|
||||
{
|
||||
git diff --name-only 2>/dev/null
|
||||
git diff --name-only --cached 2>/dev/null
|
||||
@@ -118,7 +122,10 @@ if [[ "$needs_build" == "1" ]]; then
|
||||
cd "$PROJECT_ROOT"
|
||||
new_head=$(git rev-parse HEAD 2>/dev/null || echo "unknown")
|
||||
# Recompute fingerprint of remaining dirty files (build may not change them)
|
||||
mapfile -t post_files < <(
|
||||
post_files=()
|
||||
while IFS= read -r path; do
|
||||
post_files+=("$path")
|
||||
done < <(
|
||||
{
|
||||
git diff --name-only 2>/dev/null
|
||||
git diff --name-only --cached 2>/dev/null
|
||||
@@ -165,4 +172,5 @@ fingerprint=${new_fingerprint}
|
||||
EOF
|
||||
fi
|
||||
|
||||
cd "$CALLER_PWD"
|
||||
exec "$BINARY" "$@"
|
||||
|
||||
@@ -6,3 +6,7 @@
|
||||
["gateway:docker"]
|
||||
description = "Run a standalone gateway with the bundled Docker compute driver"
|
||||
run = "bash tasks/scripts/gateway-docker.sh"
|
||||
|
||||
["gateway:vm"]
|
||||
description = "Run a standalone gateway with the bundled VM compute driver"
|
||||
run = "bash tasks/scripts/gateway-vm.sh"
|
||||
|
||||
Executable
+345
@@ -0,0 +1,345 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Start a standalone openshell-gateway backed by the VM compute driver
|
||||
# (openshell-driver-vm) for local manual testing.
|
||||
#
|
||||
# Invocation:
|
||||
# mise run gateway:vm
|
||||
#
|
||||
# Defaults:
|
||||
# - Plaintext HTTP on 127.0.0.1:18081
|
||||
# - Dedicated CLI gateway "vm-dev"
|
||||
# - Persistent gateway state (SQLite DB) under .cache/gateway-vm
|
||||
# - Per-sandbox VM driver state (rootfs + compute-driver.sock) under
|
||||
# /tmp/openshell-vm-driver-<user>-<gateway-name> so the AF_UNIX socket
|
||||
# path stays under macOS SUN_LEN
|
||||
#
|
||||
# Common overrides:
|
||||
# OPENSHELL_SERVER_PORT=18091 mise run gateway:vm
|
||||
# OPENSHELL_VM_GATEWAY_NAME=my-vm-gateway mise run gateway:vm
|
||||
# OPENSHELL_SANDBOX_NAMESPACE=my-ns mise run gateway:vm
|
||||
# OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:vm
|
||||
# mise run gateway:vm -- --gpu
|
||||
#
|
||||
# This script also writes ~/.config/openshell/active_gateway so the
|
||||
# `openshell` CLI automatically targets this gateway in subsequent shells.
|
||||
# No need to run `openshell gateway select`. Inside this repo you can
|
||||
# override per-developer with OPENSHELL_GATEWAY in `.env` (mise loads it).
|
||||
# An explicit `--gateway` / `--gateway-endpoint` flag still wins.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
PORT="${OPENSHELL_SERVER_PORT:-18081}"
|
||||
GATEWAY_NAME="${OPENSHELL_VM_GATEWAY_NAME:-vm-dev}"
|
||||
STATE_DIR="${OPENSHELL_VM_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-vm}"
|
||||
SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-vm-dev}"
|
||||
SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}"
|
||||
SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}"
|
||||
LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}"
|
||||
GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway"
|
||||
DRIVER_DIR_DEFAULT="${ROOT}/target/debug"
|
||||
DRIVER_DIR="${OPENSHELL_DRIVER_DIR:-${DRIVER_DIR_DEFAULT}}"
|
||||
COMPRESSED_DIR_DEFAULT="${ROOT}/target/vm-runtime-compressed"
|
||||
COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR_DEFAULT}}"
|
||||
VM_HOST_GATEWAY_DEFAULT="${OPENSHELL_VM_HOST_GATEWAY:-host.containers.internal}"
|
||||
GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-http://${VM_HOST_GATEWAY_DEFAULT}:${PORT}}"
|
||||
|
||||
normalize_arch() {
|
||||
case "$1" in
|
||||
x86_64|amd64) echo "amd64" ;;
|
||||
aarch64|arm64) echo "arm64" ;;
|
||||
*) echo "$1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
normalize_bool() {
|
||||
case "${1,,}" in
|
||||
1|true|yes|on) echo "true" ;;
|
||||
0|false|no|off) echo "false" ;;
|
||||
*)
|
||||
echo "ERROR: invalid boolean value '$1' (expected true/false, 1/0, yes/no, on/off)" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
port_is_in_use() {
|
||||
local port=$1
|
||||
if command -v lsof >/dev/null 2>&1; then
|
||||
lsof -nP -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1
|
||||
return $?
|
||||
fi
|
||||
if command -v nc >/dev/null 2>&1; then
|
||||
nc -z 127.0.0.1 "${port}" >/dev/null 2>&1
|
||||
return $?
|
||||
fi
|
||||
(echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1
|
||||
}
|
||||
|
||||
invoking_user() {
|
||||
if [ -n "${SUDO_USER:-}" ] && [ "${SUDO_USER}" != "root" ]; then
|
||||
printf '%s\n' "${SUDO_USER}"
|
||||
else
|
||||
id -un
|
||||
fi
|
||||
}
|
||||
|
||||
invoking_user_home() {
|
||||
local user=$1
|
||||
local home
|
||||
if [ "${user}" = "$(id -un)" ]; then
|
||||
printf '%s\n' "${HOME}"
|
||||
return
|
||||
fi
|
||||
if command -v getent >/dev/null 2>&1; then
|
||||
home="$(getent passwd "${user}" | cut -d: -f6)"
|
||||
if [ -n "${home}" ]; then
|
||||
printf '%s\n' "${home}"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
if command -v dscl >/dev/null 2>&1; then
|
||||
home="$(dscl . -read "/Users/${user}" NFSHomeDirectory 2>/dev/null | awk '{print $2}')"
|
||||
if [ -n "${home}" ]; then
|
||||
printf '%s\n' "${home}"
|
||||
return
|
||||
fi
|
||||
fi
|
||||
if [ "$(uname -s)" = "Darwin" ]; then
|
||||
printf '/Users/%s\n' "${user}"
|
||||
else
|
||||
printf '/home/%s\n' "${user}"
|
||||
fi
|
||||
}
|
||||
|
||||
gateway_config_home() {
|
||||
local user home
|
||||
user="$(invoking_user)"
|
||||
if [ -n "${SUDO_USER:-}" ] && [ "${user}" != "$(id -un)" ]; then
|
||||
home="$(invoking_user_home "${user}")"
|
||||
printf '%s\n' "${home}/.config"
|
||||
else
|
||||
printf '%s\n' "${XDG_CONFIG_HOME:-${HOME}/.config}"
|
||||
fi
|
||||
}
|
||||
|
||||
chown_invoking_user() {
|
||||
if [ -n "${SUDO_UID:-}" ] && [ -n "${SUDO_GID:-}" ]; then
|
||||
chown -R "${SUDO_UID}:${SUDO_GID}" "$@" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
register_gateway_metadata() {
|
||||
local name=$1
|
||||
local endpoint=$2
|
||||
local port=$3
|
||||
local vm_driver_state_dir=$4
|
||||
local config_home gateway_dir
|
||||
|
||||
config_home="$(gateway_config_home)"
|
||||
gateway_dir="${config_home}/openshell/gateways/${name}"
|
||||
|
||||
mkdir -p "${gateway_dir}"
|
||||
chmod 700 "${gateway_dir}" 2>/dev/null || true
|
||||
cat >"${gateway_dir}/metadata.json" <<EOF
|
||||
{
|
||||
"name": "${name}",
|
||||
"gateway_endpoint": "${endpoint}",
|
||||
"is_remote": false,
|
||||
"gateway_port": ${port},
|
||||
"auth_mode": "plaintext",
|
||||
"vm_driver_state_dir": "${vm_driver_state_dir}"
|
||||
}
|
||||
EOF
|
||||
chmod 600 "${gateway_dir}/metadata.json" 2>/dev/null || true
|
||||
chown_invoking_user "${config_home}/openshell"
|
||||
}
|
||||
|
||||
# Mirror what `openshell gateway select <name>` does: write the gateway name
|
||||
# to $XDG_CONFIG_HOME/openshell/active_gateway. The CLI picks it up as the
|
||||
# default target when neither --gateway nor OPENSHELL_GATEWAY is set.
|
||||
save_active_gateway() {
|
||||
local name=$1
|
||||
local config_home active_gateway_path
|
||||
config_home="$(gateway_config_home)"
|
||||
active_gateway_path="${config_home}/openshell/active_gateway"
|
||||
mkdir -p "$(dirname "${active_gateway_path}")"
|
||||
printf '%s' "${name}" >"${active_gateway_path}"
|
||||
chown_invoking_user "${config_home}/openshell"
|
||||
}
|
||||
|
||||
check_supervisor_cross_toolchain() {
|
||||
# The sandbox supervisor inside the guest is always Linux. On non-Linux
|
||||
# hosts (macOS) and on Linux hosts with a different arch than the guest,
|
||||
# `mise run vm:supervisor` cross-compiles via cargo-zigbuild and needs
|
||||
# the matching rustup target installed.
|
||||
local host_os host_arch guest_arch rust_target
|
||||
host_os="$(uname -s)"
|
||||
host_arch="$(uname -m)"
|
||||
guest_arch="${GUEST_ARCH:-${host_arch}}"
|
||||
case "${guest_arch}" in
|
||||
arm64|aarch64) rust_target="aarch64-unknown-linux-gnu" ;;
|
||||
x86_64|amd64) rust_target="x86_64-unknown-linux-gnu" ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
if [ "${host_os}" = "Linux" ] && [ "${host_arch}" = "${guest_arch}" ]; then
|
||||
return 0
|
||||
fi
|
||||
local missing=0
|
||||
if ! command -v cargo-zigbuild >/dev/null 2>&1; then
|
||||
echo "ERROR: cargo-zigbuild not found (required to cross-compile the guest supervisor)." >&2
|
||||
echo " Install: cargo install --locked cargo-zigbuild && brew install zig" >&2
|
||||
missing=1
|
||||
fi
|
||||
if ! rustup target list --installed 2>/dev/null | grep -qx "${rust_target}"; then
|
||||
echo "ERROR: Rust target '${rust_target}' not installed." >&2
|
||||
echo " Install: rustup target add ${rust_target}" >&2
|
||||
missing=1
|
||||
fi
|
||||
if [ "${missing}" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
VM_GPU="$(normalize_bool "${OPENSHELL_VM_GPU:-false}")"
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--gpu)
|
||||
VM_GPU="true"
|
||||
shift
|
||||
;;
|
||||
--gpu-mem-mib)
|
||||
if [ "$#" -lt 2 ]; then
|
||||
echo "ERROR: --gpu-mem-mib requires a value" >&2
|
||||
exit 2
|
||||
fi
|
||||
export OPENSHELL_VM_GPU_MEM_MIB="$2"
|
||||
shift 2
|
||||
;;
|
||||
--gpu-vcpus)
|
||||
if [ "$#" -lt 2 ]; then
|
||||
echo "ERROR: --gpu-vcpus requires a value" >&2
|
||||
exit 2
|
||||
fi
|
||||
export OPENSHELL_VM_GPU_VCPUS="$2"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
echo "Usage: mise run gateway:vm -- [--gpu] [--gpu-mem-mib MIB] [--gpu-vcpus N]"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: unknown gateway-vm option '$1'" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "${VM_GPU}" = "true" ]; then
|
||||
export OPENSHELL_VM_GPU="true"
|
||||
else
|
||||
unset OPENSHELL_VM_GPU
|
||||
fi
|
||||
|
||||
if [[ ! "${GATEWAY_NAME}" =~ ^[A-Za-z0-9._-]+$ ]]; then
|
||||
echo "ERROR: OPENSHELL_VM_GATEWAY_NAME must contain only letters, numbers, dots, underscores, or dashes" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if port_is_in_use "${PORT}"; then
|
||||
echo "ERROR: port ${PORT} is already in use; free it or set OPENSHELL_SERVER_PORT" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# AF_UNIX SUN_LEN on macOS is 104 bytes; the VM driver places
|
||||
# `compute-driver.sock` directly under VM_DRIVER_STATE_DIR, so anchor it
|
||||
# under /tmp instead of `${ROOT}/.cache` (which is typically too long on
|
||||
# macOS dev boxes with worktree paths).
|
||||
STATE_LABEL="$(printf '%s' "${GATEWAY_NAME}" | tr -cs '[:alnum:]._-' '-')"
|
||||
if [ -z "${STATE_LABEL}" ]; then
|
||||
STATE_LABEL="vm-dev"
|
||||
fi
|
||||
VM_DRIVER_STATE_DIR_DEFAULT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}/openshell-vm-driver-${USER:-user}-${STATE_LABEL}"
|
||||
VM_DRIVER_STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${VM_DRIVER_STATE_DIR_DEFAULT}}"
|
||||
|
||||
DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")"
|
||||
|
||||
# Build prerequisites: VM runtime artifacts + bundled supervisor.
|
||||
if [ ! -d "${COMPRESSED_DIR}" ] \
|
||||
|| ! find "${COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q . \
|
||||
|| [ ! -f "${COMPRESSED_DIR}/gvproxy.zst" ]; then
|
||||
echo "==> Preparing embedded VM runtime (mise run vm:setup)"
|
||||
mise run vm:setup
|
||||
fi
|
||||
|
||||
if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then
|
||||
check_supervisor_cross_toolchain
|
||||
echo "==> Building bundled VM supervisor (mise run vm:supervisor)"
|
||||
mise run vm:supervisor
|
||||
fi
|
||||
|
||||
export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${COMPRESSED_DIR}"
|
||||
|
||||
CARGO_BUILD_JOBS_ARG=()
|
||||
if [[ -n "${CARGO_BUILD_JOBS:-}" ]]; then
|
||||
CARGO_BUILD_JOBS_ARG=(-j "${CARGO_BUILD_JOBS}")
|
||||
fi
|
||||
|
||||
echo "==> Building openshell-gateway and openshell-driver-vm"
|
||||
cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \
|
||||
-p openshell-server -p openshell-driver-vm
|
||||
|
||||
if [ "$(uname -s)" = "Darwin" ]; then
|
||||
echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)"
|
||||
codesign \
|
||||
--entitlements "${ROOT}/crates/openshell-driver-vm/entitlements.plist" \
|
||||
--force \
|
||||
-s - \
|
||||
"${DRIVER_DIR}/openshell-driver-vm"
|
||||
fi
|
||||
|
||||
mkdir -p "${STATE_DIR}"
|
||||
mkdir -p "${VM_DRIVER_STATE_DIR}"
|
||||
|
||||
GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}"
|
||||
register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" "${VM_DRIVER_STATE_DIR}"
|
||||
save_active_gateway "${GATEWAY_NAME}"
|
||||
|
||||
echo "Starting standalone VM gateway..."
|
||||
echo " gateway: ${GATEWAY_NAME}"
|
||||
echo " endpoint: ${GATEWAY_ENDPOINT}"
|
||||
echo " namespace: ${SANDBOX_NAMESPACE}"
|
||||
echo " state dir: ${STATE_DIR}"
|
||||
echo " driver: ${DRIVER_DIR}/openshell-driver-vm"
|
||||
echo " driver dir: ${VM_DRIVER_STATE_DIR}"
|
||||
echo " gpu: ${VM_GPU}"
|
||||
echo " image: ${SANDBOX_IMAGE}"
|
||||
echo
|
||||
echo "Active gateway set to '${GATEWAY_NAME}'. The CLI now targets this gateway"
|
||||
echo "by default — just run \`openshell <command>\`. Override with --gateway"
|
||||
echo "or by setting OPENSHELL_GATEWAY (e.g. in .env)."
|
||||
echo
|
||||
|
||||
GATEWAY_ARGS=(
|
||||
--port "${PORT}"
|
||||
--log-level "${LOG_LEVEL}"
|
||||
--drivers vm
|
||||
--db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc"
|
||||
--sandbox-namespace "${SANDBOX_NAMESPACE}"
|
||||
--sandbox-image "${SANDBOX_IMAGE}"
|
||||
--sandbox-image-pull-policy "${SANDBOX_IMAGE_PULL_POLICY}"
|
||||
--grpc-endpoint "${GRPC_ENDPOINT}"
|
||||
--driver-dir "${DRIVER_DIR}"
|
||||
--vm-driver-state-dir "${VM_DRIVER_STATE_DIR}"
|
||||
)
|
||||
|
||||
if [ "${DISABLE_TLS}" = "true" ]; then
|
||||
GATEWAY_ARGS+=(--disable-tls)
|
||||
fi
|
||||
|
||||
exec "${GATEWAY_BIN}" "${GATEWAY_ARGS[@]}"
|
||||
Executable
+183
@@ -0,0 +1,183 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
OUTPUT_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${ROOT}/target/vm-runtime-compressed}"
|
||||
|
||||
GUEST_ARCH=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--arch)
|
||||
GUEST_ARCH="$2"
|
||||
shift 2
|
||||
;;
|
||||
--arch=*)
|
||||
GUEST_ARCH="${1#--arch=}"
|
||||
shift
|
||||
;;
|
||||
--help|-h)
|
||||
echo "Usage: $0 [--arch aarch64|x86_64]"
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "Unknown argument: $1" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "${GUEST_ARCH}" ]; then
|
||||
case "$(uname -m)" in
|
||||
aarch64|arm64) GUEST_ARCH="aarch64" ;;
|
||||
x86_64|amd64) GUEST_ARCH="x86_64" ;;
|
||||
*)
|
||||
echo "ERROR: Unsupported host architecture: $(uname -m)" >&2
|
||||
echo " Use --arch aarch64 or --arch x86_64 to override." >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
case "${GUEST_ARCH}" in
|
||||
aarch64|arm64)
|
||||
RUST_TARGET="aarch64-unknown-linux-gnu"
|
||||
;;
|
||||
x86_64|amd64)
|
||||
RUST_TARGET="x86_64-unknown-linux-gnu"
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: Unsupported guest architecture: ${GUEST_ARCH}" >&2
|
||||
echo " Supported: aarch64, x86_64" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
SUPERVISOR_BIN="${ROOT}/target/${RUST_TARGET}/release/openshell-sandbox"
|
||||
SUPERVISOR_OUTPUT="${OUTPUT_DIR}/openshell-sandbox.zst"
|
||||
|
||||
ensure_build_nofile_limit() {
|
||||
local desired="${OPENSHELL_VM_BUILD_NOFILE_LIMIT:-8192}"
|
||||
local minimum=1024
|
||||
local current=""
|
||||
local hard=""
|
||||
local target=""
|
||||
|
||||
[ "$(uname -s)" = "Darwin" ] || return 0
|
||||
command -v cargo-zigbuild >/dev/null 2>&1 || return 0
|
||||
|
||||
current="$(ulimit -n 2>/dev/null || echo "")"
|
||||
case "${current}" in
|
||||
''|*[!0-9]*)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "${current}" -ge "${desired}" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
hard="$(ulimit -Hn 2>/dev/null || echo "")"
|
||||
target="${desired}"
|
||||
case "${hard}" in
|
||||
''|unlimited|infinity)
|
||||
;;
|
||||
*[!0-9]*)
|
||||
;;
|
||||
*)
|
||||
if [ "${hard}" -lt "${target}" ]; then
|
||||
target="${hard}"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "${target}" -gt "${current}" ] && ulimit -n "${target}" 2>/dev/null; then
|
||||
echo "==> Raised open file limit for cargo-zigbuild: ${current} -> $(ulimit -n)"
|
||||
fi
|
||||
|
||||
current="$(ulimit -n 2>/dev/null || echo "${current}")"
|
||||
case "${current}" in
|
||||
''|*[!0-9]*)
|
||||
return 0
|
||||
;;
|
||||
esac
|
||||
|
||||
if [ "${current}" -lt "${desired}" ]; then
|
||||
echo "WARNING: Open file limit is ${current}; cargo-zigbuild is more reliable at ${desired}+ on macOS."
|
||||
fi
|
||||
|
||||
if [ "${current}" -lt "${minimum}" ]; then
|
||||
echo "ERROR: Open file limit (${current}) is too low for cargo-zigbuild on macOS." >&2
|
||||
echo " Run: ulimit -n ${desired}" >&2
|
||||
echo " Then re-run this script." >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
echo "==> Building openshell-sandbox supervisor bundle"
|
||||
echo " Guest arch: ${GUEST_ARCH}"
|
||||
echo " Rust target: ${RUST_TARGET}"
|
||||
echo " Output: ${SUPERVISOR_OUTPUT}"
|
||||
|
||||
mkdir -p "${OUTPUT_DIR}"
|
||||
ensure_build_nofile_limit
|
||||
|
||||
SUPERVISOR_BUILD_LOG="$(mktemp -t openshell-supervisor-build.XXXXXX.log)"
|
||||
run_supervisor_build() {
|
||||
local rustc_wrapper_mode="${1:-default}"
|
||||
local cargo_prefix=()
|
||||
|
||||
if [ "${rustc_wrapper_mode}" = "without-rustc-wrapper" ]; then
|
||||
cargo_prefix=(env -u RUSTC_WRAPPER)
|
||||
fi
|
||||
|
||||
if command -v cargo-zigbuild >/dev/null 2>&1; then
|
||||
"${cargo_prefix[@]}" cargo zigbuild --release -p openshell-sandbox --target "${RUST_TARGET}" \
|
||||
--manifest-path "${ROOT}/Cargo.toml"
|
||||
else
|
||||
echo " cargo-zigbuild not found, falling back to cargo build..."
|
||||
"${cargo_prefix[@]}" cargo build --release -p openshell-sandbox --target "${RUST_TARGET}" \
|
||||
--manifest-path "${ROOT}/Cargo.toml"
|
||||
fi
|
||||
}
|
||||
|
||||
print_build_failure() {
|
||||
echo "ERROR: supervisor build failed. Full output:" >&2
|
||||
cat "${SUPERVISOR_BUILD_LOG}" >&2
|
||||
echo " (log saved at ${SUPERVISOR_BUILD_LOG})" >&2
|
||||
}
|
||||
|
||||
if run_supervisor_build >"${SUPERVISOR_BUILD_LOG}" 2>&1; then
|
||||
tail -5 "${SUPERVISOR_BUILD_LOG}"
|
||||
rm -f "${SUPERVISOR_BUILD_LOG}"
|
||||
else
|
||||
status=$?
|
||||
if [ -n "${RUSTC_WRAPPER:-}" ] && grep -Eq 'sccache: encountered fatal error|Too many open files|os error 24' "${SUPERVISOR_BUILD_LOG}"; then
|
||||
echo "WARNING: supervisor build failed through RUSTC_WRAPPER=${RUSTC_WRAPPER}; retrying without RUSTC_WRAPPER." >&2
|
||||
: >"${SUPERVISOR_BUILD_LOG}"
|
||||
if run_supervisor_build without-rustc-wrapper >"${SUPERVISOR_BUILD_LOG}" 2>&1; then
|
||||
tail -5 "${SUPERVISOR_BUILD_LOG}"
|
||||
rm -f "${SUPERVISOR_BUILD_LOG}"
|
||||
else
|
||||
status=$?
|
||||
print_build_failure
|
||||
exit "${status}"
|
||||
fi
|
||||
else
|
||||
print_build_failure
|
||||
exit "${status}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ ! -f "${SUPERVISOR_BIN}" ]; then
|
||||
echo "ERROR: supervisor binary not found at ${SUPERVISOR_BIN}" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
zstd -19 -T0 -f "${SUPERVISOR_BIN}" -o "${SUPERVISOR_OUTPUT}"
|
||||
|
||||
echo "==> Bundled supervisor ready"
|
||||
echo " Binary: $(du -sh "${SUPERVISOR_BIN}" | cut -f1)"
|
||||
echo " Compressed: $(du -sh "${SUPERVISOR_OUTPUT}" | cut -f1)"
|
||||
@@ -21,6 +21,7 @@ set -euo pipefail
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${SCRIPT_DIR}/_lib.sh"
|
||||
ROOT="$(vm_lib_root)"
|
||||
CLI_BIN="${ROOT}/scripts/bin/openshell"
|
||||
|
||||
FROM_SOURCE="${FROM_SOURCE:-0}"
|
||||
|
||||
@@ -126,6 +127,6 @@ echo ""
|
||||
echo "==> Setup complete!"
|
||||
echo " Compressed artifacts in: ${OUTPUT_DIR}"
|
||||
echo ""
|
||||
echo "Next steps:"
|
||||
echo " mise run vm:rootfs --base # build rootfs (requires Docker)"
|
||||
echo " mise run gateway:vm # start openshell-gateway with the VM driver"
|
||||
echo "After starting the gateway:"
|
||||
echo " ${CLI_BIN} status"
|
||||
echo " ${CLI_BIN} sandbox create --name vm-test --from ubuntu:24.04"
|
||||
|
||||
+8
-5
@@ -5,21 +5,20 @@
|
||||
#
|
||||
# Workflow:
|
||||
# mise run vm:setup # one-time: download pre-built runtime (~30s)
|
||||
# mise run vm:supervisor # build the bundled sandbox supervisor
|
||||
# mise run gateway:vm # start openshell-gateway with the VM driver
|
||||
# # (defined in tasks/gateway.toml)
|
||||
# mise run vm # build + run the standalone openshell-vm microVM
|
||||
# mise run vm:clean # wipe everything and start over
|
||||
#
|
||||
# See crates/openshell-driver-vm/README.md for the `gateway:vm` flow and
|
||||
# See tasks/gateway.toml for `gateway:vm`,
|
||||
# crates/openshell-driver-vm/README.md for the VM driver workflow, and
|
||||
# crates/openshell-vm/README.md for the standalone microVM path.
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# Main Commands
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
|
||||
["gateway:vm"]
|
||||
description = "Build openshell-gateway + openshell-driver-vm and start the gateway with the VM driver (pass -- --gpu for GPU support)"
|
||||
run = "crates/openshell-driver-vm/start.sh"
|
||||
|
||||
[vm]
|
||||
description = "Build and run the standalone openshell-vm microVM"
|
||||
depends = ["build:docker:gateway"]
|
||||
@@ -38,6 +37,10 @@ run = [
|
||||
description = "One-time setup: download (or build) the VM runtime"
|
||||
run = "tasks/scripts/vm/vm-setup.sh"
|
||||
|
||||
["vm:supervisor"]
|
||||
description = "Build the bundled openshell-sandbox supervisor for openshell-driver-vm"
|
||||
run = "tasks/scripts/vm/build-supervisor-bundle.sh"
|
||||
|
||||
["vm:rootfs"]
|
||||
description = "Build the VM rootfs tarball (use -- --base for lightweight)"
|
||||
run = "tasks/scripts/vm/build-rootfs-tarball.sh"
|
||||
|
||||
Reference in New Issue
Block a user