From 2c4d8977c0ff871ca6ea869382e8a9cea993c414 Mon Sep 17 00:00:00 2001 From: Drew Newberry Date: Mon, 27 Apr 2026 20:59:45 -0700 Subject: [PATCH] wip: gateway:vm refactor + driver-scoped ssh handshake check --- architecture/custom-vm-runtime.md | 11 +- crates/openshell-driver-vm/README.md | 67 ++-- .../scripts/openshell-vm-sandbox-init.sh | 52 ++- crates/openshell-driver-vm/src/driver.rs | 65 +++- crates/openshell-driver-vm/src/runtime.rs | 16 +- crates/openshell-driver-vm/start.sh | 305 ------------------ crates/openshell-server/src/compute/vm.rs | 11 +- crates/openshell-server/src/lib.rs | 75 +++-- e2e/rust/e2e-vm.sh | 9 +- tasks/gateway.toml | 4 + tasks/scripts/gateway-vm.sh | 225 +++++++++++++ tasks/vm.toml | 8 +- 12 files changed, 441 insertions(+), 407 deletions(-) delete mode 100755 crates/openshell-driver-vm/start.sh create mode 100755 tasks/scripts/gateway-vm.sh diff --git a/architecture/custom-vm-runtime.md b/architecture/custom-vm-runtime.md index 18594d4af..3c721ae50 100644 --- a/architecture/custom-vm-runtime.md +++ b/architecture/custom-vm-runtime.md @@ -220,8 +220,8 @@ graph LR The `vm-runtime-.tar.zst` artifact is consumed by `openshell-driver-vm`'s `build.rs`, which embeds the library set into the binary via `include_bytes!()`. Setting `OPENSHELL_VM_RUNTIME_COMPRESSED_DIR` -at build time (wired up by `crates/openshell-driver-vm/start.sh`) points the -build at the staged artifacts. +at build time (wired up by `tasks/scripts/gateway-vm.sh`, registered as +`mise run gateway:vm`) points the build at the staged artifacts. ## Kernel Config Fragment @@ -315,14 +315,15 @@ cross-compiled via osxcross (no macOS runner needed for the binary build — only for the kernel build). macOS driver binaries produced via osxcross are not codesigned. Development -builds are signed automatically by `crates/openshell-driver-vm/start.sh`; a -packaged release needs signing in CI. +builds are signed automatically by `tasks/scripts/gateway-vm.sh` +(registered as `mise run gateway:vm`); a packaged release needs signing in +CI. ## Rollout Strategy 1. Custom runtime is embedded by default when building `openshell-driver-vm` with `OPENSHELL_VM_RUNTIME_COMPRESSED_DIR` set (wired up by - `crates/openshell-driver-vm/start.sh`). + `tasks/scripts/gateway-vm.sh`). 2. The sandbox init script validates kernel capabilities at boot and fails fast if missing. 3. For development, override with `OPENSHELL_VM_RUNTIME_DIR` to use a local diff --git a/crates/openshell-driver-vm/README.md b/crates/openshell-driver-vm/README.md index def0ae4db..d42dea442 100644 --- a/crates/openshell-driver-vm/README.md +++ b/crates/openshell-driver-vm/README.md @@ -35,10 +35,15 @@ Sandbox guests execute `/opt/openshell/bin/openshell-sandbox` as PID 1 inside th mise run gateway:vm ``` -First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:supervisor` builds the bundled guest supervisor. Subsequent runs are cached. To keep the Unix socket path under macOS `SUN_LEN`, `mise run gateway:vm` and `start.sh` default the state dir to `/tmp/openshell-vm-driver-dev-$USER-/` (SQLite DB + per-sandbox rootfs + `compute-driver.sock`) unless `OPENSHELL_VM_DRIVER_STATE_DIR` is set. -By default the wrapper names the gateway after the repo directory, writes `OPENSHELL_GATEWAY=` into `.env`, and writes plaintext local gateway metadata under `~/.config/openshell/gateways//metadata.json` so repo-local `scripts/bin/openshell status` and `sandbox create` resolve to the VM gateway without an extra `gateway select`. When running under `sudo`, the wrapper uses `sudo -u $SUDO_USER` for the registration so the config is written under the invoking user's home directory. Re-runs are idempotent. -If neither `OPENSHELL_SERVER_PORT` nor `GATEWAY_PORT` is set, the wrapper picks a random free local port once and appends `GATEWAY_PORT=` to `.env`. Later runs reuse that port through `mise`'s env loading. If you set `OPENSHELL_SERVER_PORT` explicitly, the wrapper uses it for that run and still fails fast on conflicts. -It also exports `OPENSHELL_DRIVER_DIR=$PWD/target/debug` before starting the gateway so local dev runs use the freshly built `openshell-driver-vm` instead of an older installed copy from `~/.local/libexec/openshell` or `/usr/local/libexec`. +First run takes a few minutes while `mise run vm:setup` stages libkrun/libkrunfw/gvproxy and `mise run vm:supervisor` builds the bundled guest supervisor. Subsequent runs are cached. + +By default `mise run gateway:vm`: + +- Listens on plaintext HTTP at `127.0.0.1:18081`. +- Registers the CLI gateway `vm-dev` by writing `~/.config/openshell/gateways/vm-dev/metadata.json`. It does not modify the workspace `.env`. +- Persists the gateway SQLite DB under `.cache/gateway-vm/gateway.db`. +- Places the VM driver state (per-sandbox rootfs + `compute-driver.sock`) under `/tmp/openshell-vm-driver-$USER-vm-dev/` so the AF_UNIX socket path stays under macOS `SUN_LEN`. +- Passes `--driver-dir $PWD/target/debug` so the freshly built `openshell-driver-vm` is used instead of an older installed copy from `~/.local/libexec/openshell` or `/usr/local/libexec`. For GPU passthrough (VFIO), pass `-- --gpu` and run with root privileges: @@ -48,45 +53,38 @@ sudo -E env "PATH=$PATH" mise run gateway:vm -- --gpu See [`architecture/vm-gpu-sandbox-guide.md`](../../architecture/vm-gpu-sandbox-guide.md) for full GPU prerequisites and usage. -Override via environment: +Point the CLI at the gateway with one of: ```shell -OPENSHELL_SERVER_PORT=9090 \ -OPENSHELL_SSH_HANDSHAKE_SECRET=$(openssl rand -hex 32) \ -crates/openshell-driver-vm/start.sh +openshell --gateway vm-dev status +openshell gateway select vm-dev # then plain `openshell ` ``` -If you want to pin the project port instead of using the `.env` default: +Override defaults via environment: ```shell -GATEWAY_PORT=28080 mise run gateway:vm -``` +# custom port (fails fast if in use) +OPENSHELL_SERVER_PORT=18091 mise run gateway:vm -If you want a custom state-dir suffix instead of the repo-name default, set `OPENSHELL_VM_INSTANCE`: - -```shell -GATEWAY_PORT=28081 \ -OPENSHELL_VM_INSTANCE=feature-a \ -mise run gateway:vm -``` - -If you want a custom CLI gateway name instead of the repo directory, set `OPENSHELL_VM_GATEWAY_NAME`: - -```shell -GATEWAY_PORT=28082 \ +# custom CLI gateway name + namespace OPENSHELL_VM_GATEWAY_NAME=vm-feature-a \ +OPENSHELL_SANDBOX_NAMESPACE=vm-feature-a \ mise run gateway:vm + +# custom sandbox image +OPENSHELL_SANDBOX_IMAGE=ghcr.io/example/sandbox:latest mise run gateway:vm ``` Teardown: ```shell -rm -rf /tmp/openshell-vm-driver-dev-$USER-$(basename "$PWD" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//') +rm -rf /tmp/openshell-vm-driver-$USER-vm-dev .cache/gateway-vm +rm -rf "${XDG_CONFIG_HOME:-$HOME/.config}/openshell/gateways/vm-dev" ``` ## Manual equivalent -If you want to drive the launch yourself instead of using `start.sh`: +If you want to drive the launch yourself instead of using `mise run gateway:vm` (i.e. `tasks/scripts/gateway-vm.sh`): ```shell # 1. Stage runtime artifacts + supervisor bundle into target/vm-runtime-compressed/ @@ -103,18 +101,17 @@ codesign \ --force -s - target/debug/openshell-driver-vm # 4. Start the gateway with the VM driver -mkdir -p /tmp/openshell-vm-driver-dev-$USER-port-8080 +mkdir -p /tmp/openshell-vm-driver-$USER-vm-dev .cache/gateway-vm target/debug/openshell-gateway \ --drivers vm \ --disable-tls \ - --database-url sqlite:/tmp/openshell-vm-driver-dev-$USER-port-8080/openshell.db \ + --db-url "sqlite:.cache/gateway-vm/gateway.db?mode=rwc" \ --driver-dir $PWD/target/debug \ + --sandbox-namespace vm-dev \ --sandbox-image \ - --grpc-endpoint http://host.containers.internal:8080 \ - --ssh-handshake-secret dev-vm-driver-secret \ - --ssh-gateway-host 127.0.0.1 \ - --ssh-gateway-port 8080 \ - --vm-driver-state-dir /tmp/openshell-vm-driver-dev-$USER-port-8080 + --grpc-endpoint http://host.containers.internal:18081 \ + --port 18081 \ + --vm-driver-state-dir /tmp/openshell-vm-driver-$USER-vm-dev ``` The gateway resolves `openshell-driver-vm` in this order: `--driver-dir`, conventional install locations (`~/.local/libexec/openshell`, `/usr/local/libexec/openshell`, `/usr/local/libexec`), then a sibling of the gateway binary. @@ -124,7 +121,7 @@ The gateway resolves `openshell-driver-vm` in this order: `--driver-dir`, conven | Flag | Env var | Default | Purpose | |---|---|---|---| | `--drivers vm` | `OPENSHELL_DRIVERS` | `kubernetes` | Select the VM compute driver. | -| `--grpc-endpoint URL` | `OPENSHELL_GRPC_ENDPOINT` | — | Required. URL the sandbox guest calls back to. Use a host alias that resolves to the gateway's host from inside the VM (`host.containers.internal` comes from gvproxy DNS; the guest init script also seeds `host.openshell.internal` to `192.168.127.1`). | +| `--grpc-endpoint URL` | `OPENSHELL_GRPC_ENDPOINT` | — | Required. URL the sandbox guest dials to reach the gateway. Use `http://host.containers.internal:` (or `host.docker.internal` / `host.openshell.internal`) so traffic flows through gvproxy's host-loopback NAT (HostIP `192.168.127.254` → host `127.0.0.1`). Loopback URLs like `http://127.0.0.1:` are rewritten automatically by the driver. The bare gateway IP (`192.168.127.1`) only carries gvproxy's own services and will not reach host-bound ports. | | `--vm-driver-state-dir DIR` | `OPENSHELL_VM_DRIVER_STATE_DIR` | `target/openshell-vm-driver` | Per-sandbox rootfs, console logs, and the `compute-driver.sock` UDS. | | `--driver-dir DIR` | `OPENSHELL_DRIVER_DIR` | unset | Override the directory searched for `openshell-driver-vm`. | | `--vm-driver-vcpus N` | `OPENSHELL_VM_DRIVER_VCPUS` | `2` | vCPUs per sandbox. | @@ -154,7 +151,7 @@ Raise log verbosity for both processes: ```shell RUST_LOG=openshell_server=debug,openshell_driver_vm=debug \ - crates/openshell-driver-vm/start.sh + mise run gateway:vm ``` The VM guest's serial console is appended to `//console.log`. The `compute-driver.sock` lives at `/compute-driver.sock`; the gateway removes it on clean shutdown via `ManagedDriverProcess::drop`. @@ -177,4 +174,4 @@ The VM guest's serial console is appended to `//console.l ## TODOs - The gateway still configures the driver via CLI args; this will move to a gRPC bootstrap call so the driver interface is uniform across backends. See the `TODO(driver-abstraction)` notes in `crates/openshell-server/src/lib.rs` and `crates/openshell-server/src/compute/vm.rs`. -- macOS codesigning is handled by `start.sh`; a packaged release would need signing in CI. +- macOS codesigning is handled by `tasks/scripts/gateway-vm.sh`; a packaged release would need signing in CI. diff --git a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh index 8c38abdb4..063a75032 100644 --- a/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh +++ b/crates/openshell-driver-vm/scripts/openshell-vm-sandbox-init.sh @@ -15,7 +15,20 @@ if [ -f /srv/openshell-env.sh ]; then fi BOOT_START=$(date +%s%3N 2>/dev/null || date +%s) +# gvisor-tap-vsock subnet layout: +# 192.168.127.1 — gateway: gvproxy's DNS / DHCP / HTTP API. Does NOT +# proxy arbitrary host ports. +# 192.168.127.254 — host-loopback: NAT-rewritten to host's 127.0.0.1 by +# gvproxy's TCP/UDP/ICMP forwarder. Use this address +# (or any of the host.* hostnames below) to reach a +# service the host is listening on. +# The host.containers.internal / host.docker.internal DNS records served +# by gvproxy's embedded resolver point at 192.168.127.254. We mirror that +# in /etc/hosts so the supervisor can reach the gateway even when +# gvproxy's DNS is not in resolv.conf (e.g. DHCP failed and we fell +# back to 8.8.8.8). GVPROXY_GATEWAY_IP="192.168.127.1" +GVPROXY_HOST_LOOPBACK_IP="192.168.127.254" GATEWAY_IP="$GVPROXY_GATEWAY_IP" # Parse kernel cmdline for GPU and TAP networking parameters @@ -96,11 +109,19 @@ tcp_probe() { } ensure_host_gateway_aliases() { - # Pin every host-gateway alias to the gvproxy gateway IP via /etc/hosts so the - # supervisor can reach the OpenShell server even when gvproxy's built-in DNS - # is not in resolv.conf (e.g. when DHCP fails and we fall back to 8.8.8.8). + # Seed /etc/hosts with the well-known gvproxy hostnames so the supervisor + # can reach the OpenShell server even when gvproxy's built-in DNS is not + # in resolv.conf (e.g. when DHCP fails and we fall back to 8.8.8.8). + # + # Critical distinction: host.* aliases point at the gvproxy *host-loopback* + # IP (192.168.127.254), not the gateway IP (192.168.127.1). Only the + # host-loopback IP carries NAT rewriting to the host's 127.0.0.1 — the + # gateway IP only listens on gvproxy's own service ports (DNS:53, DHCP, + # HTTP API:80). Pinning host.containers.internal to the gateway IP + # silently breaks guest→host port reachability for arbitrary ports. local hosts_tmp="/tmp/openshell-hosts.$$" - local aliases="host.openshell.internal host.containers.internal host.docker.internal gateway.containers.internal" + local host_aliases="host.openshell.internal host.containers.internal host.docker.internal" + local gateway_aliases="gateway.containers.internal" local filter='(^|[[:space:]])(host\.openshell\.internal|host\.containers\.internal|host\.docker\.internal|gateway\.containers\.internal)([[:space:]]|$)' if [ -f /etc/hosts ]; then @@ -109,7 +130,18 @@ ensure_host_gateway_aliases() { : > "$hosts_tmp" fi - printf '%s %s\n' "$GATEWAY_IP" "$aliases" >> "$hosts_tmp" + # In TAP/GPU mode, GATEWAY_IP is overridden to VM_NET_GW (the host-side + # of the TAP), and the gateway is reachable directly there. In gvproxy + # mode, host.openshell.internal etc. need GVPROXY_HOST_LOOPBACK_IP + # (192.168.127.254) which is gvproxy's host-NAT entry, while + # gateway.containers.internal points at the gvproxy gateway itself. + if [ "${GATEWAY_IP}" = "${GVPROXY_GATEWAY_IP}" ]; then + printf '%s %s\n' "$GVPROXY_HOST_LOOPBACK_IP" "$host_aliases" >> "$hosts_tmp" + printf '%s %s\n' "$GVPROXY_GATEWAY_IP" "$gateway_aliases" >> "$hosts_tmp" + else + # TAP networking: gateway and host are both reachable at GATEWAY_IP. + printf '%s %s %s\n' "$GATEWAY_IP" "$host_aliases" "$gateway_aliases" >> "$hosts_tmp" + fi cat "$hosts_tmp" > /etc/hosts rm -f "$hosts_tmp" } @@ -134,7 +166,15 @@ rewrite_openshell_endpoint_if_needed() { return 0 fi - for candidate in host.openshell.internal host.containers.internal host.docker.internal "$GATEWAY_IP"; do + # Probe candidates in preference order. Hostnames first for informative + # log output, then a bare IP as a final safety net. In gvproxy mode the + # bare IP is the host-loopback (192.168.127.254). In TAP/GPU mode it's + # the TAP host gateway. + local fallback_ip="$GVPROXY_HOST_LOOPBACK_IP" + if [ "${GATEWAY_IP}" != "${GVPROXY_GATEWAY_IP}" ]; then + fallback_ip="$GATEWAY_IP" + fi + for candidate in host.openshell.internal host.containers.internal host.docker.internal "$fallback_ip"; do if [ "$candidate" = "$host" ]; then continue fi diff --git a/crates/openshell-driver-vm/src/driver.rs b/crates/openshell-driver-vm/src/driver.rs index 7d4dbb9f9..6e2b25656 100644 --- a/crates/openshell-driver-vm/src/driver.rs +++ b/crates/openshell-driver-vm/src/driver.rs @@ -52,8 +52,36 @@ const DRIVER_NAME: &str = "openshell-driver-vm"; const WATCH_BUFFER: usize = 256; const DEFAULT_VCPUS: u8 = 2; const DEFAULT_MEM_MIB: u32 = 2048; +/// gvproxy gateway IP — runs DNS, DHCP, and the gvproxy HTTP API. Does **not** +/// proxy arbitrary host ports. const GVPROXY_GATEWAY_IP: &str = "192.168.127.1"; +/// gvproxy host-loopback IP — gvproxy's TCP/UDP/ICMP forwarder NAT-rewrites +/// this destination to the host's `127.0.0.1` and dials out from the host +/// process. This is the only address that transparently reaches host-bound +/// services without explicit `expose` rules. +/// +/// See gvisor-tap-vsock `cmd/gvproxy/config.go` (default NAT entry +/// `HostIP -> 127.0.0.1`) and `pkg/services/forwarder/tcp.go` (NAT lookup +/// before `net.Dial`). +/// +/// Code paths route via `GVPROXY_HOST_LOOPBACK_ALIAS` (DNS / /etc/hosts) +/// instead so logs stay readable; this constant is kept for documentation +/// and parity with the guest init script. +#[allow(dead_code)] +const GVPROXY_HOST_LOOPBACK_IP: &str = "192.168.127.254"; const OPENSHELL_HOST_GATEWAY_ALIAS: &str = "host.openshell.internal"; +/// Hostname gvproxy resolves (via its embedded DNS) to the host-loopback IP. +/// +/// We rewrite loopback URLs to this hostname rather than the bare IP because: +/// * the guest init script seeds /etc/hosts with the same mapping, so it +/// resolves even when gvproxy's DNS is not in resolv.conf; +/// * keeping a recognisable hostname makes log messages clearer than a bare +/// 192.168.127.254 reference; +/// * `host.docker.internal` works the same way for Docker-flavoured tooling. +/// +/// Both names ultimately route through the gvproxy NAT path on +/// `GVPROXY_HOST_LOOPBACK_IP` — they do **not** go through the gateway IP. +const GVPROXY_HOST_LOOPBACK_ALIAS: &str = "host.containers.internal"; const GUEST_SSH_SOCKET_PATH: &str = "/run/openshell/ssh.sock"; const GUEST_TLS_DIR: &str = "/opt/openshell/tls"; const GUEST_TLS_CA_PATH: &str = "/opt/openshell/tls/ca.crt"; @@ -1660,6 +1688,25 @@ fn merged_environment(sandbox: &Sandbox) -> HashMap { environment } +/// Rewrites loopback host references in a gateway URL to a hostname the guest +/// can reach via gvproxy. +/// +/// The driver receives the gateway endpoint from `--openshell-endpoint`, which +/// in local/dev/e2e setups is typically `http://127.0.0.1:`. That URL is +/// useless inside the guest because the guest's loopback interface is its own, +/// not the host's. Inside the guest we need a name that gvproxy will translate +/// into the host's loopback address. +/// +/// We rewrite to `host.containers.internal`, which gvproxy's embedded DNS resolves +/// to the host-loopback IP `192.168.127.254`. gvproxy installs a default NAT entry +/// rewriting that destination to the host's `127.0.0.1` and dialing out from the +/// host process, so any port the host is listening on becomes reachable. The +/// gateway IP `192.168.127.1` does **not** do this — it only listens on gvproxy's +/// own service ports (DNS, DHCP, HTTP API). The guest init script also seeds the +/// hostname in `/etc/hosts` so resolution works even if gvproxy's DNS isn't in +/// resolv.conf (e.g. when DHCP fails). +/// +/// Non-loopback URLs are returned unchanged. fn guest_visible_openshell_endpoint(endpoint: &str) -> String { let Ok(mut url) = Url::parse(endpoint) else { return endpoint.to_string(); @@ -1672,7 +1719,7 @@ fn guest_visible_openshell_endpoint(endpoint: &str) -> String { None => false, }; - if should_rewrite && url.set_host(Some(GVPROXY_GATEWAY_IP)).is_ok() { + if should_rewrite && url.set_host(Some(GVPROXY_HOST_LOOPBACK_ALIAS)).is_ok() { return url.to_string(); } @@ -2181,7 +2228,7 @@ mod tests { let env = build_guest_environment(&sandbox, &config, None); assert!(env.contains(&"HOME=/root".to_string())); assert!(env.contains(&format!( - "OPENSHELL_ENDPOINT=http://{GVPROXY_GATEWAY_IP}:8080/" + "OPENSHELL_ENDPOINT=http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/" ))); assert!(env.contains(&"OPENSHELL_SANDBOX_ID=sandbox-123".to_string())); assert!(env.contains(&format!( @@ -2223,18 +2270,18 @@ mod tests { } #[test] - fn guest_visible_openshell_endpoint_rewrites_loopback_hosts_to_gvproxy_gateway() { + fn guest_visible_openshell_endpoint_rewrites_loopback_hosts_to_gvproxy_host_alias() { assert_eq!( guest_visible_openshell_endpoint("http://127.0.0.1:8080"), - format!("http://{GVPROXY_GATEWAY_IP}:8080/") + format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/") ); assert_eq!( guest_visible_openshell_endpoint("http://localhost:8080"), - format!("http://{GVPROXY_GATEWAY_IP}:8080/") + format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080/") ); assert_eq!( guest_visible_openshell_endpoint("https://[::1]:8443"), - format!("https://{GVPROXY_GATEWAY_IP}:8443/") + format!("https://{GVPROXY_HOST_LOOPBACK_ALIAS}:8443/") ); } @@ -2247,8 +2294,10 @@ mod tests { format!("http://{OPENSHELL_HOST_GATEWAY_ALIAS}:8080") ); assert_eq!( - guest_visible_openshell_endpoint("http://host.containers.internal:8080"), - "http://host.containers.internal:8080" + guest_visible_openshell_endpoint(&format!( + "http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080" + )), + format!("http://{GVPROXY_HOST_LOOPBACK_ALIAS}:8080") ); assert_eq!( guest_visible_openshell_endpoint(&format!("http://{GVPROXY_GATEWAY_IP}:8080")), diff --git a/crates/openshell-driver-vm/src/runtime.rs b/crates/openshell-driver-vm/src/runtime.rs index c063da10b..a7c9afcea 100644 --- a/crates/openshell-driver-vm/src/runtime.rs +++ b/crates/openshell-driver-vm/src/runtime.rs @@ -705,13 +705,15 @@ fn run_libkrun_vm(config: &VmLaunchConfig) -> Result<(), String> { // talks to on boot (IPs 192.168.127.1 / .2, defaults for // gvisor-tap-vsock); // * the host-facing gateway identity the guest uses for callbacks: - // the init script seeds `/etc/hosts` with - // `host.openshell.internal` pointing at 192.168.127.1 while - // leaving gvproxy's legacy `host.containers.internal` / - // `host.docker.internal` DNS answers intact, which is how the guest's - // `rewrite_openshell_endpoint_if_needed` probe reaches the host - // gateway when the bare loopback address doesn't resolve from - // inside the VM. + // gvproxy installs a default NAT entry rewriting `192.168.127.254` + // (the subnet's HostIP) to the host's `127.0.0.1`, and serves + // `host.containers.internal` / `host.docker.internal` / + // `host.openshell.internal` in its embedded DNS pointing at that + // same HostIP. The guest init script seeds /etc/hosts with the + // same mapping so the supervisor reaches the host gateway even + // when gvproxy's DNS isn't in resolv.conf. The gateway IP + // (192.168.127.1) is NOT a host-loopback proxy — it only listens + // on its own service ports (DNS:53, DHCP, HTTP API:80). // // That network plane is also what the sandbox supervisor's // per-sandbox netns (veth pair + iptables, see diff --git a/crates/openshell-driver-vm/start.sh b/crates/openshell-driver-vm/start.sh deleted file mode 100755 index c81cc22ed..000000000 --- a/crates/openshell-driver-vm/start.sh +++ /dev/null @@ -1,305 +0,0 @@ -#!/usr/bin/env bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -set -euo pipefail - -# Under sudo, PATH is reset and user-local tools (mise, cargo) disappear. -# Restore the invoking user's tool directories so mise and its shims work. -if [ -n "${SUDO_USER:-}" ]; then - _sudo_home=$(getent passwd "${SUDO_USER}" | cut -d: -f6) - for _p in "${_sudo_home}/.local/bin" "${_sudo_home}/.local/share/mise/shims" "${_sudo_home}/.cargo/bin"; do - [ -d "${_p}" ] && PATH="${_p}:${PATH}" - done - export PATH -fi - -ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -source "${ROOT}/crates/openshell-vm/pins.env" 2>/dev/null || true -CLI_BIN="${ROOT}/scripts/bin/openshell" -ENV_FILE="${ROOT}/.env" -COMPRESSED_DIR_DEFAULT="${ROOT}/target/vm-runtime-compressed" -COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR_DEFAULT}}" -SERVER_PORT_REQUESTED="${OPENSHELL_SERVER_PORT:-${GATEWAY_PORT:-}}" -SERVER_PORT="${SERVER_PORT_REQUESTED:-}" -STATE_DIR_ROOT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}" -VM_HOST_GATEWAY_DEFAULT="${OPENSHELL_VM_HOST_GATEWAY:-host.containers.internal}" -DRIVER_DIR_DEFAULT="${ROOT}/target/debug" -DRIVER_DIR="${OPENSHELL_DRIVER_DIR:-${DRIVER_DIR_DEFAULT}}" - -normalize_name() { - printf '%s' "$1" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9-]/-/g' | sed 's/--*/-/g' | sed 's/^-//;s/-$//' -} - -for arg in "$@"; do - if [ "${arg}" = "--gpu" ]; then - export OPENSHELL_VM_GPU=true - break - fi -done - -has_env_key() { - local key=$1 - [ -f "${ENV_FILE}" ] || return 1 - grep -Eq "^[[:space:]]*(export[[:space:]]+)?${key}=" "${ENV_FILE}" -} - -append_env_if_missing() { - local key=$1 - local value=$2 - if has_env_key "${key}"; then - return - fi - if [ -f "${ENV_FILE}" ] && [ -s "${ENV_FILE}" ]; then - if [ "$(tail -c1 "${ENV_FILE}" | wc -l)" -eq 0 ]; then - printf "\n" >>"${ENV_FILE}" - fi - fi - printf "%s=%s\n" "${key}" "${value}" >>"${ENV_FILE}" -} - -upsert_env_key() { - local key=$1 - local value=$2 - local tmp_file - - tmp_file="$(mktemp "${ENV_FILE}.tmp.XXXXXX")" - if [ -f "${ENV_FILE}" ]; then - awk -v key="${key}" -v value="${value}" ' - BEGIN { updated = 0 } - $0 ~ "^[[:space:]]*(export[[:space:]]+)?" key "=" { - if (!updated) { - print key "=" value - updated = 1 - } - next - } - { print } - END { - if (!updated) { - print key "=" value - } - } - ' "${ENV_FILE}" >"${tmp_file}" - else - printf "%s=%s\n" "${key}" "${value}" >"${tmp_file}" - fi - mv "${tmp_file}" "${ENV_FILE}" -} - -normalize_bool() { - case "${1,,}" in - 1|true|yes|on) echo "true" ;; - 0|false|no|off) echo "false" ;; - *) - echo "invalid boolean value '$1' (expected true/false, 1/0, yes/no, on/off)" >&2 - exit 1 - ;; - esac -} - -port_is_in_use() { - local port=$1 - if command -v lsof >/dev/null 2>&1; then - lsof -nP -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1 - return $? - fi - - if command -v nc >/dev/null 2>&1; then - nc -z 127.0.0.1 "${port}" >/dev/null 2>&1 - return $? - fi - - (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 -} - -pick_random_port() { - local lower=20000 - local upper=60999 - local attempts=256 - local port - - for _ in $(seq 1 "${attempts}"); do - port=$((RANDOM % (upper - lower + 1) + lower)) - if ! port_is_in_use "${port}"; then - echo "${port}" - return 0 - fi - done - - echo "ERROR: could not find a free port after ${attempts} attempts." >&2 - return 1 -} - -check_supervisor_cross_toolchain() { - # The sandbox supervisor inside the guest is always Linux. On non-Linux - # hosts (macOS) and on Linux hosts with a different arch than the guest, - # we cross-compile via cargo-zigbuild and need the matching rustup target. - local host_os host_arch guest_arch rust_target - host_os="$(uname -s)" - host_arch="$(uname -m)" - guest_arch="${GUEST_ARCH:-${host_arch}}" - case "${guest_arch}" in - arm64|aarch64) rust_target="aarch64-unknown-linux-gnu" ;; - x86_64|amd64) rust_target="x86_64-unknown-linux-gnu" ;; - *) return 0 ;; - esac - if [ "${host_os}" = "Linux" ] && [ "${host_arch}" = "${guest_arch}" ]; then - return 0 - fi - local missing=0 - if ! command -v cargo-zigbuild >/dev/null 2>&1; then - echo "ERROR: cargo-zigbuild not found (required to cross-compile the guest supervisor)." >&2 - echo " Install: cargo install --locked cargo-zigbuild && brew install zig" >&2 - missing=1 - fi - if ! rustup target list --installed 2>/dev/null | grep -qx "${rust_target}"; then - echo "ERROR: Rust target '${rust_target}' not installed." >&2 - echo " Install: rustup target add ${rust_target}" >&2 - missing=1 - fi - if [ "${missing}" -ne 0 ]; then - exit 1 - fi -} - -build_gateway_and_driver() { - local rustc_wrapper_mode="${1:-default}" - local cargo_prefix=() - - if [ "${rustc_wrapper_mode}" = "without-rustc-wrapper" ]; then - cargo_prefix=(env -u RUSTC_WRAPPER) - fi - - "${cargo_prefix[@]}" cargo build -p openshell-server -p openshell-driver-vm -} - -if [ -n "${SERVER_PORT_REQUESTED}" ]; then - if port_is_in_use "${SERVER_PORT}"; then - echo "ERROR: requested gateway port ${SERVER_PORT} is already in use." >&2 - echo " Update .env GATEWAY_PORT or override it for one run:" >&2 - echo " OPENSHELL_SERVER_PORT= mise run gateway:vm" >&2 - exit 1 - fi -else - SERVER_PORT="$(pick_random_port)" - append_env_if_missing "GATEWAY_PORT" "${SERVER_PORT}" -fi - -GATEWAY_NAME_DEFAULT="$(basename "${ROOT}")" -GATEWAY_NAME="${OPENSHELL_VM_GATEWAY_NAME:-${GATEWAY_NAME_DEFAULT}}" -if [ -z "${GATEWAY_NAME}" ]; then - GATEWAY_NAME="openshell" -fi - -# Keep the driver socket path under AF_UNIX SUN_LEN on macOS. -STATE_LABEL_RAW="${OPENSHELL_VM_INSTANCE:-$(normalize_name "${GATEWAY_NAME}")}" -STATE_LABEL="$(printf '%s' "${STATE_LABEL_RAW}" | tr -cs '[:alnum:]._-' '-')" -if [ -z "${STATE_LABEL}" ]; then - STATE_LABEL="gateway" -fi -STATE_DIR_DEFAULT="${STATE_DIR_ROOT}/openshell-vm-driver-dev-${USER:-user}-${STATE_LABEL}" -STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${STATE_DIR_DEFAULT}}" -DB_PATH_DEFAULT="${STATE_DIR}/openshell.db" -LOCAL_GATEWAY_ENDPOINT_DEFAULT="http://127.0.0.1:${SERVER_PORT}" -LOCAL_GATEWAY_ENDPOINT="${OPENSHELL_VM_LOCAL_GATEWAY_ENDPOINT:-${LOCAL_GATEWAY_ENDPOINT_DEFAULT}}" - -export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${COMPRESSED_DIR}" -export OPENSHELL_GATEWAY="${GATEWAY_NAME}" - -upsert_env_key "OPENSHELL_GATEWAY" "${GATEWAY_NAME}" - -mkdir -p "${STATE_DIR}" - -if [ ! -d "${COMPRESSED_DIR}" ] || ! find "${COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q . || [ ! -f "${COMPRESSED_DIR}/gvproxy.zst" ]; then - echo "==> Preparing embedded VM runtime" - mise run vm:setup -fi - -if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then - check_supervisor_cross_toolchain - echo "==> Building bundled VM supervisor" - mise run vm:supervisor -fi - -echo "==> Building gateway and VM compute driver" -if ! build_gateway_and_driver; then - if [ -n "${RUSTC_WRAPPER:-}" ]; then - echo "WARNING: gateway/driver build failed through RUSTC_WRAPPER=${RUSTC_WRAPPER}; retrying without RUSTC_WRAPPER." >&2 - build_gateway_and_driver without-rustc-wrapper - else - exit 1 - fi -fi - -if [ "$(uname -s)" = "Darwin" ]; then - echo "==> Codesigning VM compute driver" - codesign \ - --entitlements "${ROOT}/crates/openshell-driver-vm/entitlements.plist" \ - --force \ - -s - \ - "${ROOT}/target/debug/openshell-driver-vm" -fi - -export OPENSHELL_DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")" -export OPENSHELL_DB_URL="${OPENSHELL_DB_URL:-sqlite:${DB_PATH_DEFAULT}}" -export OPENSHELL_DRIVERS="${OPENSHELL_DRIVERS:-vm}" -export OPENSHELL_DRIVER_DIR="${DRIVER_DIR}" -export OPENSHELL_SERVER_PORT="${SERVER_PORT}" -export OPENSHELL_GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-http://${VM_HOST_GATEWAY_DEFAULT}:${SERVER_PORT}}" -export OPENSHELL_SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-}}" -export OPENSHELL_SSH_GATEWAY_HOST="${OPENSHELL_SSH_GATEWAY_HOST:-127.0.0.1}" -export OPENSHELL_SSH_GATEWAY_PORT="${OPENSHELL_SSH_GATEWAY_PORT:-${SERVER_PORT}}" -export OPENSHELL_SSH_HANDSHAKE_SECRET="${OPENSHELL_SSH_HANDSHAKE_SECRET:-dev-vm-driver-secret}" -export OPENSHELL_VM_DRIVER_STATE_DIR="${STATE_DIR}" - -# Resolve the VM runtime directory (contains vmlinux, virtiofsd, etc.) -# so the child --internal-run-vm process can find it under sudo. -if [ -z "${OPENSHELL_VM_RUNTIME_DIR:-}" ]; then - _candidate="${HOME}/.local/share/openshell/vm-runtime/0.0.0" - if [ -n "${SUDO_USER:-}" ]; then - _sudo_home=$(getent passwd "${SUDO_USER}" | cut -d: -f6) - _candidate="${_sudo_home}/.local/share/openshell/vm-runtime/0.0.0" - fi - if [ -f "${_candidate}/vmlinux" ]; then - export OPENSHELL_VM_RUNTIME_DIR="${_candidate}" - fi -fi - -# Write gateway metadata to the invoking user's config dir, even under sudo. -# When running under sudo, $HOME points at /root and XDG_CONFIG_HOME may also -# be inherited from the root env; fall back to SUDO_USER's home directory so -# repo-local `scripts/bin/openshell` reads the same config. -if [ -n "${SUDO_USER:-}" ]; then - _sudo_home=$(getent passwd "${SUDO_USER}" | cut -d: -f6) - GATEWAY_CONFIG_BASE="${_sudo_home}/.config" -else - GATEWAY_CONFIG_BASE="${XDG_CONFIG_HOME:-${HOME}/.config}" -fi -GATEWAY_METADATA_DIR="${GATEWAY_CONFIG_BASE}/openshell/gateways/${GATEWAY_NAME}" -mkdir -p "${GATEWAY_METADATA_DIR}" -cat >"${GATEWAY_METADATA_DIR}/metadata.json" </dev/null || true -fi - -echo "==> Gateway config" -echo " Name: ${GATEWAY_NAME}" -echo " Endpoint: ${LOCAL_GATEWAY_ENDPOINT}" -echo " .env: OPENSHELL_GATEWAY=${GATEWAY_NAME}" -echo " .env: GATEWAY_PORT=${SERVER_PORT}" -echo " Driver: ${OPENSHELL_DRIVER_DIR}/openshell-driver-vm" -echo " Image: ${OPENSHELL_SANDBOX_IMAGE}" -echo " Status: ${CLI_BIN} status" -echo " Create: ${CLI_BIN} sandbox create --name vm-test --from ubuntu:24.04" - -echo "==> Starting OpenShell server with VM compute driver" -exec "${ROOT}/target/debug/openshell-gateway" diff --git a/crates/openshell-server/src/compute/vm.rs b/crates/openshell-server/src/compute/vm.rs index e5d2880ec..23b728060 100644 --- a/crates/openshell-server/src/compute/vm.rs +++ b/crates/openshell-server/src/compute/vm.rs @@ -310,9 +310,14 @@ pub async fn spawn( if !vm_config.default_image.trim().is_empty() { command.arg("--default-image").arg(&vm_config.default_image); } - command - .arg("--ssh-handshake-secret") - .arg(&config.ssh_handshake_secret); + // Only forward the handshake secret when one is configured. The VM + // driver does not consume it, but accepts it for parity with the + // Kubernetes/Podman drivers; passing an empty value is noise. + if !config.ssh_handshake_secret.is_empty() { + command + .arg("--ssh-handshake-secret") + .arg(&config.ssh_handshake_secret); + } command .arg("--ssh-handshake-skew-secs") .arg(config.ssh_handshake_skew_secs.to_string()); diff --git a/crates/openshell-server/src/lib.rs b/crates/openshell-server/src/lib.rs index 95c18fdda..6c59e9822 100644 --- a/crates/openshell-server/src/lib.rs +++ b/crates/openshell-server/src/lib.rs @@ -148,12 +148,10 @@ pub async fn run_server( if database_url.is_empty() { return Err(Error::config("database_url is required")); } - let driver = configured_compute_driver(&config)?; - if config.ssh_handshake_secret.is_empty() && driver != ComputeDriverKind::Docker { - return Err(Error::config( - "ssh_handshake_secret is required. Set --ssh-handshake-secret or OPENSHELL_SSH_HANDSHAKE_SECRET", - )); - } + // The ssh_handshake_secret check is driver-scoped: the Kubernetes and + // Podman drivers inject the secret into sandbox env, while the VM and + // Docker drivers do not consume it. The check is enforced per-driver + // in `build_compute_runtime`. let store = Arc::new(Store::connect(database_url).await?); @@ -368,32 +366,35 @@ async fn build_compute_runtime( info!(driver = %driver, "Using compute driver"); match driver { - ComputeDriverKind::Kubernetes => ComputeRuntime::new_kubernetes( - KubernetesComputeConfig { - namespace: config.sandbox_namespace.clone(), - default_image: config.sandbox_image.clone(), - image_pull_policy: config.sandbox_image_pull_policy.clone(), - grpc_endpoint: config.grpc_endpoint.clone(), - // Filesystem path to the supervisor's Unix-socket SSH daemon. - // The path lives in a root-only directory so only the - // supervisor can connect; the gateway reaches it through the - // RelayStream bridge, not directly. Override via - // `sandbox_ssh_socket_path` in the config for deployments - // where multiple supervisors share a filesystem. - ssh_socket_path: config.sandbox_ssh_socket_path.clone(), - ssh_handshake_secret: config.ssh_handshake_secret.clone(), - ssh_handshake_skew_secs: config.ssh_handshake_skew_secs, - client_tls_secret_name: config.client_tls_secret_name.clone(), - host_gateway_ip: config.host_gateway_ip.clone(), - }, - store, - sandbox_index, - sandbox_watch_bus, - tracing_log_bus, - supervisor_sessions.clone(), - ) - .await - .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))), + ComputeDriverKind::Kubernetes => { + require_ssh_handshake_secret(config)?; + ComputeRuntime::new_kubernetes( + KubernetesComputeConfig { + namespace: config.sandbox_namespace.clone(), + default_image: config.sandbox_image.clone(), + image_pull_policy: config.sandbox_image_pull_policy.clone(), + grpc_endpoint: config.grpc_endpoint.clone(), + // Filesystem path to the supervisor's Unix-socket SSH daemon. + // The path lives in a root-only directory so only the + // supervisor can connect; the gateway reaches it through + // the RelayStream bridge, not directly. Override via + // `sandbox_ssh_socket_path` in the config for deployments + // where multiple supervisors share a filesystem. + ssh_socket_path: config.sandbox_ssh_socket_path.clone(), + ssh_handshake_secret: config.ssh_handshake_secret.clone(), + ssh_handshake_skew_secs: config.ssh_handshake_skew_secs, + client_tls_secret_name: config.client_tls_secret_name.clone(), + host_gateway_ip: config.host_gateway_ip.clone(), + }, + store, + sandbox_index, + sandbox_watch_bus, + tracing_log_bus, + supervisor_sessions.clone(), + ) + .await + .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) + } ComputeDriverKind::Docker => ComputeRuntime::new_docker( config.clone(), docker_config.clone(), @@ -420,6 +421,7 @@ async fn build_compute_runtime( .map_err(|e| Error::execution(format!("failed to create compute runtime: {e}"))) } ComputeDriverKind::Podman => { + require_ssh_handshake_secret(config)?; let socket_path = std::env::var("OPENSHELL_PODMAN_SOCKET") .ok() .filter(|s| !s.is_empty()) @@ -471,6 +473,15 @@ async fn build_compute_runtime( } } +fn require_ssh_handshake_secret(config: &Config) -> Result<()> { + if config.ssh_handshake_secret.is_empty() { + return Err(Error::config( + "ssh_handshake_secret is required for this driver. Set --ssh-handshake-secret or OPENSHELL_SSH_HANDSHAKE_SECRET", + )); + } + Ok(()) +} + fn configured_compute_driver(config: &Config) -> Result { match config.compute_drivers.as_slice() { [] => Err(Error::config( diff --git a/e2e/rust/e2e-vm.sh b/e2e/rust/e2e-vm.sh index 5990d8db6..551f9b41e 100755 --- a/e2e/rust/e2e-vm.sh +++ b/e2e/rust/e2e-vm.sh @@ -167,13 +167,20 @@ echo "==> Starting openshell-gateway on 127.0.0.1:${HOST_PORT} (state: ${RUN_STA # `~/.local/libexec/openshell/openshell-driver-vm` when present # (install-vm.sh installs there), which silently shadows development # builds — a subtle source of stale-binary bugs in e2e runs. +# --grpc-endpoint is the URL the VM driver passes into each guest as +# OPENSHELL_ENDPOINT. The supervisor inside the VM dials this address. +# Use `host.containers.internal` rather than `127.0.0.1` so gvproxy's +# host-loopback proxy carries the connection — gvproxy's bare gateway IP +# (192.168.127.1) does NOT forward arbitrary host ports. The driver also +# rewrites loopback URLs to this hostname as a safety net, so this matches +# what the guest will actually see and aligns with `tasks/scripts/gateway-vm.sh`. "${GATEWAY_BIN}" \ --drivers vm \ --disable-tls \ --disable-gateway-auth \ --db-url 'sqlite::memory:' \ --port "${HOST_PORT}" \ - --grpc-endpoint "http://127.0.0.1:${HOST_PORT}" \ + --grpc-endpoint "http://host.containers.internal:${HOST_PORT}" \ --ssh-handshake-secret "${SSH_HANDSHAKE_SECRET}" \ --driver-dir "${ROOT}/target/debug" \ --vm-driver-state-dir "${RUN_STATE_DIR}" \ diff --git a/tasks/gateway.toml b/tasks/gateway.toml index 3f7a684d2..1df07cb95 100644 --- a/tasks/gateway.toml +++ b/tasks/gateway.toml @@ -6,3 +6,7 @@ ["gateway:docker"] description = "Run a standalone gateway with the bundled Docker compute driver" run = "bash tasks/scripts/gateway-docker.sh" + +["gateway:vm"] +description = "Run a standalone gateway with the bundled VM compute driver" +run = "bash tasks/scripts/gateway-vm.sh" diff --git a/tasks/scripts/gateway-vm.sh b/tasks/scripts/gateway-vm.sh new file mode 100755 index 000000000..f0623a46c --- /dev/null +++ b/tasks/scripts/gateway-vm.sh @@ -0,0 +1,225 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Start a standalone openshell-gateway backed by the VM compute driver +# (openshell-driver-vm) for local manual testing. +# +# Defaults: +# - Plaintext HTTP on 127.0.0.1:18081 +# - Dedicated CLI gateway "vm-dev" +# - Persistent gateway state (SQLite DB) under .cache/gateway-vm +# - Per-sandbox VM driver state (rootfs + compute-driver.sock) under +# /tmp/openshell-vm-driver-- so the AF_UNIX socket +# path stays under macOS SUN_LEN +# +# Common overrides: +# OPENSHELL_SERVER_PORT=18091 mise run gateway:vm +# OPENSHELL_VM_GATEWAY_NAME=my-vm-gateway mise run gateway:vm +# OPENSHELL_SANDBOX_NAMESPACE=my-ns mise run gateway:vm +# OPENSHELL_SANDBOX_IMAGE=ghcr.io/... mise run gateway:vm +# +# After the gateway is running, point the CLI at it with either: +# openshell --gateway vm-dev +# openshell gateway select vm-dev # then plain `openshell ` + +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +PORT="${OPENSHELL_SERVER_PORT:-18081}" +GATEWAY_NAME="${OPENSHELL_VM_GATEWAY_NAME:-vm-dev}" +STATE_DIR="${OPENSHELL_VM_GATEWAY_STATE_DIR:-${ROOT}/.cache/gateway-vm}" +SANDBOX_NAMESPACE="${OPENSHELL_SANDBOX_NAMESPACE:-vm-dev}" +SANDBOX_IMAGE="${OPENSHELL_SANDBOX_IMAGE:-${COMMUNITY_SANDBOX_IMAGE:-ghcr.io/nvidia/openshell-community/sandboxes/base:latest}}" +SANDBOX_IMAGE_PULL_POLICY="${OPENSHELL_SANDBOX_IMAGE_PULL_POLICY:-IfNotPresent}" +LOG_LEVEL="${OPENSHELL_LOG_LEVEL:-info}" +GATEWAY_BIN="${ROOT}/target/debug/openshell-gateway" +DRIVER_DIR_DEFAULT="${ROOT}/target/debug" +DRIVER_DIR="${OPENSHELL_DRIVER_DIR:-${DRIVER_DIR_DEFAULT}}" +COMPRESSED_DIR_DEFAULT="${ROOT}/target/vm-runtime-compressed" +COMPRESSED_DIR="${OPENSHELL_VM_RUNTIME_COMPRESSED_DIR:-${COMPRESSED_DIR_DEFAULT}}" +VM_HOST_GATEWAY_DEFAULT="${OPENSHELL_VM_HOST_GATEWAY:-host.containers.internal}" +GRPC_ENDPOINT="${OPENSHELL_GRPC_ENDPOINT:-http://${VM_HOST_GATEWAY_DEFAULT}:${PORT}}" + +normalize_arch() { + case "$1" in + x86_64|amd64) echo "amd64" ;; + aarch64|arm64) echo "arm64" ;; + *) echo "$1" ;; + esac +} + +normalize_bool() { + case "${1,,}" in + 1|true|yes|on) echo "true" ;; + 0|false|no|off) echo "false" ;; + *) + echo "ERROR: invalid boolean value '$1' (expected true/false, 1/0, yes/no, on/off)" >&2 + exit 2 + ;; + esac +} + +port_is_in_use() { + local port=$1 + if command -v lsof >/dev/null 2>&1; then + lsof -nP -iTCP:"${port}" -sTCP:LISTEN >/dev/null 2>&1 + return $? + fi + if command -v nc >/dev/null 2>&1; then + nc -z 127.0.0.1 "${port}" >/dev/null 2>&1 + return $? + fi + (echo >/dev/tcp/127.0.0.1/"${port}") >/dev/null 2>&1 +} + +register_gateway_metadata() { + local name=$1 + local endpoint=$2 + local port=$3 + local config_home gateway_dir + + config_home="${XDG_CONFIG_HOME:-${HOME}/.config}" + gateway_dir="${config_home}/openshell/gateways/${name}" + + mkdir -p "${gateway_dir}" + cat >"${gateway_dir}/metadata.json" </dev/null 2>&1; then + echo "ERROR: cargo-zigbuild not found (required to cross-compile the guest supervisor)." >&2 + echo " Install: cargo install --locked cargo-zigbuild && brew install zig" >&2 + missing=1 + fi + if ! rustup target list --installed 2>/dev/null | grep -qx "${rust_target}"; then + echo "ERROR: Rust target '${rust_target}' not installed." >&2 + echo " Install: rustup target add ${rust_target}" >&2 + missing=1 + fi + if [ "${missing}" -ne 0 ]; then + exit 1 + fi +} + +if [[ ! "${GATEWAY_NAME}" =~ ^[A-Za-z0-9._-]+$ ]]; then + echo "ERROR: OPENSHELL_VM_GATEWAY_NAME must contain only letters, numbers, dots, underscores, or dashes" >&2 + exit 2 +fi + +if port_is_in_use "${PORT}"; then + echo "ERROR: port ${PORT} is already in use; free it or set OPENSHELL_SERVER_PORT" >&2 + exit 2 +fi + +# AF_UNIX SUN_LEN on macOS is 104 bytes; the VM driver places +# `compute-driver.sock` directly under VM_DRIVER_STATE_DIR, so anchor it +# under /tmp instead of `${ROOT}/.cache` (which is typically too long on +# macOS dev boxes with worktree paths). +STATE_LABEL="$(printf '%s' "${GATEWAY_NAME}" | tr -cs '[:alnum:]._-' '-')" +if [ -z "${STATE_LABEL}" ]; then + STATE_LABEL="vm-dev" +fi +VM_DRIVER_STATE_DIR_DEFAULT="${OPENSHELL_VM_DRIVER_STATE_ROOT:-/tmp}/openshell-vm-driver-${USER:-user}-${STATE_LABEL}" +VM_DRIVER_STATE_DIR="${OPENSHELL_VM_DRIVER_STATE_DIR:-${VM_DRIVER_STATE_DIR_DEFAULT}}" + +DISABLE_TLS="$(normalize_bool "${OPENSHELL_DISABLE_TLS:-true}")" + +# Build prerequisites: VM runtime artifacts + bundled supervisor. +if [ ! -d "${COMPRESSED_DIR}" ] \ + || ! find "${COMPRESSED_DIR}" -maxdepth 1 -name 'libkrun*.zst' | grep -q . \ + || [ ! -f "${COMPRESSED_DIR}/gvproxy.zst" ]; then + echo "==> Preparing embedded VM runtime (mise run vm:setup)" + mise run vm:setup +fi + +if [ ! -f "${COMPRESSED_DIR}/openshell-sandbox.zst" ]; then + check_supervisor_cross_toolchain + echo "==> Building bundled VM supervisor (mise run vm:supervisor)" + mise run vm:supervisor +fi + +export OPENSHELL_VM_RUNTIME_COMPRESSED_DIR="${COMPRESSED_DIR}" + +CARGO_BUILD_JOBS_ARG=() +if [[ -n "${CARGO_BUILD_JOBS:-}" ]]; then + CARGO_BUILD_JOBS_ARG=(-j "${CARGO_BUILD_JOBS}") +fi + +echo "==> Building openshell-gateway and openshell-driver-vm" +cargo build ${CARGO_BUILD_JOBS_ARG[@]+"${CARGO_BUILD_JOBS_ARG[@]}"} \ + -p openshell-server -p openshell-driver-vm + +if [ "$(uname -s)" = "Darwin" ]; then + echo "==> Codesigning openshell-driver-vm (Hypervisor entitlement)" + codesign \ + --entitlements "${ROOT}/crates/openshell-driver-vm/entitlements.plist" \ + --force \ + -s - \ + "${DRIVER_DIR}/openshell-driver-vm" +fi + +mkdir -p "${STATE_DIR}" +mkdir -p "${VM_DRIVER_STATE_DIR}" + +GATEWAY_ENDPOINT="http://127.0.0.1:${PORT}" +register_gateway_metadata "${GATEWAY_NAME}" "${GATEWAY_ENDPOINT}" "${PORT}" + +echo "Starting standalone VM gateway..." +echo " gateway: ${GATEWAY_NAME}" +echo " endpoint: ${GATEWAY_ENDPOINT}" +echo " namespace: ${SANDBOX_NAMESPACE}" +echo " state dir: ${STATE_DIR}" +echo " driver: ${DRIVER_DIR}/openshell-driver-vm" +echo " driver dir: ${VM_DRIVER_STATE_DIR}" +echo " image: ${SANDBOX_IMAGE}" +echo +echo "Point the CLI at this gateway with one of:" +echo " openshell --gateway ${GATEWAY_NAME} status" +echo " openshell gateway select ${GATEWAY_NAME}" +echo + +GATEWAY_ARGS=( + --port "${PORT}" + --log-level "${LOG_LEVEL}" + --drivers vm + --db-url "sqlite:${STATE_DIR}/gateway.db?mode=rwc" + --sandbox-namespace "${SANDBOX_NAMESPACE}" + --sandbox-image "${SANDBOX_IMAGE}" + --sandbox-image-pull-policy "${SANDBOX_IMAGE_PULL_POLICY}" + --grpc-endpoint "${GRPC_ENDPOINT}" + --driver-dir "${DRIVER_DIR}" + --vm-driver-state-dir "${VM_DRIVER_STATE_DIR}" +) + +if [ "${DISABLE_TLS}" = "true" ]; then + GATEWAY_ARGS+=(--disable-tls) +fi + +exec "${GATEWAY_BIN}" "${GATEWAY_ARGS[@]}" diff --git a/tasks/vm.toml b/tasks/vm.toml index 410416000..e9eb22561 100644 --- a/tasks/vm.toml +++ b/tasks/vm.toml @@ -7,20 +7,18 @@ # mise run vm:setup # one-time: download pre-built runtime (~30s) # mise run vm:supervisor # build the bundled sandbox supervisor # mise run gateway:vm # start openshell-gateway with the VM driver +# # (defined in tasks/gateway.toml) # mise run vm # build + run the standalone openshell-vm microVM # mise run vm:clean # wipe everything and start over # -# See crates/openshell-driver-vm/README.md for the `gateway:vm` flow and +# See tasks/gateway.toml for `gateway:vm`, +# crates/openshell-driver-vm/README.md for the VM driver workflow, and # crates/openshell-vm/README.md for the standalone microVM path. # ═══════════════════════════════════════════════════════════════════════════ # Main Commands # ═══════════════════════════════════════════════════════════════════════════ -["gateway:vm"] -description = "Build openshell-gateway + openshell-driver-vm and start the gateway with the VM driver (pass -- --gpu for GPU support)" -run = "crates/openshell-driver-vm/start.sh" - [vm] description = "Build and run the standalone openshell-vm microVM" depends = ["build:docker:gateway"]