#!/usr/bin/env bash # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Build the current checkout, run its gateway on the host or in a disposable # Nix test guest, and execute one named host-side E2E suite against that gateway. set -Eeuo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # shellcheck disable=SC1091 source "${ROOT}/e2e/support/gateway-common.sh" # shellcheck disable=SC1091 source "${ROOT}/tasks/scripts/build-env.sh" e2e_preserve_mise_dirs usage() { cat <<'EOF' Usage: e2e/run.sh [--vm DISTRO] [--with CONFIG ...] \ --gateway-config PATH --suite NAME Options: --vm DISTRO Run the gateway in a Nix test guest --with CONFIG Apply a Nix test-guest configuration; repeatable --gateway-config PATH Fully resolved gateway TOML --suite NAME Rust suite at e2e/rust/tests/NAME.rs -h, --help Show this help Omit --vm and --with to run the gateway on the host. Supplying --with without --vm selects Fedora for the Podman driver and Ubuntu otherwise. Set OPENSHELL_E2E_KEEP=1 to retain state. EOF } die() { echo "ERROR: $*" >&2 exit 2 } require_value() { local option=$1 local count=$2 local value=${3:-} if [ "${count}" -lt 2 ] || [ -z "${value}" ]; then die "${option} requires a value" fi case "${value}" in --*) die "${option} requires a value" ;; esac } resolve_file() { local path=$1 if [ ! -f "${path}" ]; then return 1 fi python3 - "${path}" <<'PY' import os import sys print(os.path.realpath(sys.argv[1])) PY } catalog_has_entry() { local catalog=$1 local section=$2 local name=$3 printf '%s\n' "${catalog}" | awk -v wanted_section="${section}:" -v wanted_name="${name}" ' $0 == wanted_section { in_section = 1 next } /^[^[:space:]]/ { in_section = 0 } in_section && $0 == " " wanted_name { found = 1 } END { exit(found ? 0 : 1) } ' } vm= gateway_config= suite_name= with_configurations=() while [ "$#" -gt 0 ]; do case "$1" in --vm) require_value "$1" "$#" "${2:-}" vm=$2 shift 2 ;; --with) require_value "$1" "$#" "${2:-}" with_configurations+=("$2") shift 2 ;; --gateway-config) require_value "$1" "$#" "${2:-}" gateway_config=$2 shift 2 ;; --suite) require_value "$1" "$#" "${2:-}" suite_name=$2 shift 2 ;; -h | --help) usage exit 0 ;; *) die "unknown argument: $1" ;; esac done if [ -z "${gateway_config}" ]; then die "--gateway-config is required" fi if [ -z "${suite_name}" ]; then die "--suite is required" fi if ! command -v python3 >/dev/null 2>&1; then die "python3 is required" fi gateway_config_source=${gateway_config} if ! gateway_config="$(resolve_file "${gateway_config_source}")"; then die "gateway config does not exist: ${gateway_config_source}" fi gateway_driver="$(python3 -c ' import sys, tomllib gateway = tomllib.load(open(sys.argv[1], "rb"))["openshell"]["gateway"] driver = gateway.get("compute_driver") if not driver: raise SystemExit("gateway config must explicitly select a compute driver") print(driver) ' "${gateway_config}")" if [[ ! ${suite_name} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then die "suite name must contain only lowercase letters, digits, and hyphens: ${suite_name}" fi suite_path="${ROOT}/e2e/rust/tests/${suite_name}.rs" if [ ! -f "${suite_path}" ]; then die "unknown suite: ${suite_name}" fi mode=host if [ -n "${vm}" ] || [ "${#with_configurations[@]}" -gt 0 ]; then mode=vm if [ -z "${vm}" ]; then if [ "${gateway_driver}" = podman ]; then vm=fedora else vm=ubuntu fi fi fi if [ "${mode}" = vm ]; then if [[ ! ${vm} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then die "invalid VM distro name: ${vm}" fi for configuration in "${with_configurations[@]}"; do if [[ ! ${configuration} =~ ^[a-z0-9][a-z0-9-]*$ ]]; then die "invalid VM configuration name: ${configuration}" fi done if [ "${gateway_driver}" = podman ] && [ "${vm}" = ubuntu ]; then die "the Ubuntu 24.04 guest lacks the Podman 5 pasta helper required for sandbox callbacks; use --vm fedora --with podman" fi if ! command -v nix >/dev/null 2>&1; then die "Nix is required for VM mode" fi if ! command -v base64 >/dev/null 2>&1; then die "base64 is required for VM mode" fi if ! vm_catalog="$(cd "${ROOT}" && nix run .#test-guest -- --list)"; then die "failed to read the Nix test-guest catalog" fi if ! catalog_has_entry "${vm_catalog}" Distros "${vm}"; then die "unknown VM distro in the Nix test-guest catalog: ${vm}" fi for configuration in "${with_configurations[@]}"; do if ! catalog_has_entry "${vm_catalog}" Configurations "${configuration}"; then die "unknown VM configuration in the Nix test-guest catalog: ${configuration}" fi done fi gateway_ready_timeout=${OPENSHELL_E2E_GATEWAY_READY_TIMEOUT:-600} if [[ ! ${gateway_ready_timeout} =~ ^[1-9][0-9]*$ ]]; then die "OPENSHELL_E2E_GATEWAY_READY_TIMEOUT must be a positive integer" fi if ! command -v mise >/dev/null 2>&1; then die "mise is required to build OpenShell" fi if ! command -v openssl >/dev/null 2>&1; then die "OpenSSL is required to generate sandbox JWT keys" fi case "$(uname -m)" in x86_64 | amd64) linux_musl_target=x86_64-unknown-linux-musl linux_gateway_rust_target=x86_64-unknown-linux-gnu linux_gateway_zig_target=x86_64-unknown-linux-gnu.2.28 ;; aarch64 | arm64) linux_musl_target=aarch64-unknown-linux-musl linux_gateway_rust_target=aarch64-unknown-linux-gnu linux_gateway_zig_target=aarch64-unknown-linux-gnu.2.28 ;; *) die "unsupported host architecture: $(uname -m)" ;; esac cargo_jobs=() if [ -n "${CARGO_BUILD_JOBS:-}" ]; then cargo_jobs=(-j "${CARGO_BUILD_JOBS}") fi cd "${ROOT}" target_dir="$(e2e_cargo_target_dir "${ROOT}" mise x -- cargo)" ensure_build_nofile_limit echo "==> Building native host openshell CLI" mise x -- cargo build "${cargo_jobs[@]}" -p openshell-cli --bin openshell host_cli_bin="${target_dir}/debug/openshell" echo "==> Preparing ${linux_musl_target} build target" mise x -- rustup target add "${linux_musl_target}" >/dev/null echo "==> Building Linux openshell-sandbox (${linux_musl_target})" mise x -- cargo zigbuild "${cargo_jobs[@]}" \ --release \ --target "${linux_musl_target}" \ -p openshell-sandbox \ --bin openshell-sandbox linux_sandbox_bin="${target_dir}/${linux_musl_target}/release/openshell-sandbox" echo "==> Building Linux openshell-supervisor (${linux_musl_target})" mise x -- cargo zigbuild "${cargo_jobs[@]}" \ --release \ --target "${linux_musl_target}" \ -p openshell-supervisor \ --bin openshell-supervisor linux_supervisor_bin="${target_dir}/${linux_musl_target}/release/openshell-supervisor" host_gateway_bin= guest_gateway_bin= if [ "${mode}" = host ]; then echo "==> Building native host openshell-gateway" mise x -- cargo build "${cargo_jobs[@]}" \ -p openshell-gateway \ --bin openshell-gateway \ --features vendored-z3 host_gateway_bin="${target_dir}/debug/openshell-gateway" else echo "==> Building Linux openshell-gateway (${linux_gateway_zig_target})" ( eval "$( "${ROOT}/tasks/scripts/setup-zig-cc-wrapper.sh" \ "${linux_gateway_zig_target}" \ "${linux_gateway_zig_target}" \ "${target_dir}/zig-gnu-wrapper/e2e" )" mise x -- cargo zigbuild "${cargo_jobs[@]}" \ --release \ --target "${linux_gateway_zig_target}" \ -p openshell-gateway \ --bin openshell-gateway \ --features vendored-z3 ) guest_gateway_bin="${target_dir}/${linux_gateway_rust_target}/release/openshell-gateway" fi expected_binaries=("${host_cli_bin}" "${linux_sandbox_bin}" "${linux_supervisor_bin}") if [ "${mode}" = host ]; then expected_binaries+=("${host_gateway_bin}") else expected_binaries+=("${guest_gateway_bin}") fi for binary in "${expected_binaries[@]}"; do if [ ! -x "${binary}" ]; then echo "ERROR: expected built binary at ${binary}" >&2 exit 1 fi done run_parent="${ROOT}/.cache/openshell-e2e/runs" mkdir -p "${run_parent}" run_dir="$(mktemp -d "${run_parent%/}/run.XXXXXX")" if ! command -v tar >/dev/null 2>&1; then die "tar is required to package the runtime images" fi sandbox_runtime_image=localhost/openshell/sandbox:e2e-vm sandbox_runtime_rootfs="${run_dir}/sandbox-runtime-rootfs" sandbox_runtime_archive="${run_dir}/sandbox-runtime.tar" mkdir -p "${sandbox_runtime_rootfs}" install -m 0555 "${linux_sandbox_bin}" "${sandbox_runtime_rootfs}/openshell-sandbox" tar -C "${sandbox_runtime_rootfs}" -cf "${sandbox_runtime_archive}" openshell-sandbox supervisor_image=localhost/openshell/supervisor:e2e-vm supervisor_rootfs="${run_dir}/supervisor-rootfs" supervisor_archive="${run_dir}/supervisor.tar" mkdir -p "${supervisor_rootfs}" install -m 0555 "${linux_supervisor_bin}" "${supervisor_rootfs}/openshell-supervisor" "${ROOT}/tasks/scripts/verify-static-binary.sh" "${supervisor_rootfs}/openshell-supervisor" mkdir -p "${supervisor_rootfs}/etc/ssl/certs" if [ -f /etc/ssl/certs/ca-certificates.crt ]; then install -m 0444 /etc/ssl/certs/ca-certificates.crt \ "${supervisor_rootfs}/etc/ssl/certs/ca-certificates.crt" else die "/etc/ssl/certs/ca-certificates.crt is required to package the supervisor image" fi tar -C "${supervisor_rootfs}" -cf "${supervisor_archive}" \ openshell-supervisor etc/ssl/certs/ca-certificates.crt child_pid= runtime_log= keep=0 if [ "${OPENSHELL_E2E_KEEP:-0}" = 1 ]; then keep=1 fi start_child() { local working_dir=$1 local log_path=$2 shift 2 ( cd "${working_dir}" exec python3 -c \ 'import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])' \ "$@" ) >"${log_path}" 2>&1 & child_pid=$! } # Invoked by the EXIT trap through cleanup. # shellcheck disable=SC2329 stop_child() { local pid=$1 local signal_target="-${pid}" if [ -z "${pid}" ] || ! kill -0 "${pid}" 2>/dev/null; then return fi kill -TERM -- "${signal_target}" 2>/dev/null || true for _ in $(seq 1 30); do if ! kill -0 "${pid}" 2>/dev/null; then break fi sleep 1 done if kill -0 "${pid}" 2>/dev/null; then kill -KILL -- "${signal_target}" 2>/dev/null || true fi wait "${pid}" 2>/dev/null || true } # Invoked by EXIT, INT, and TERM traps. # shellcheck disable=SC2329 cleanup() { local status=$? trap - EXIT INT TERM stop_child "${child_pid}" if [ "${status}" -ne 0 ] && [ -n "${runtime_log}" ] && [ -f "${runtime_log}" ]; then echo "=== ${mode} gateway log ===" >&2 cat "${runtime_log}" >&2 echo "=== end ${mode} gateway log ===" >&2 fi if [ "${keep}" -eq 1 ]; then echo "Kept E2E runner state at ${run_dir}" >&2 else rm -rf "${run_dir}" fi exit "${status}" } trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM jwt_source_dir="${run_dir}/gateway-jwt" host_runtime_dir= if [ "${mode}" = host ]; then host_runtime_dir="${run_dir}/host-runtime" jwt_source_dir="${host_runtime_dir}/.cache/openshell-e2e/gateway-jwt" fi e2e_generate_gateway_jwt "${jwt_source_dir}" host_port="$(e2e_pick_port)" guest_port= if [ "${mode}" = vm ]; then guest_port=8080 fi export XDG_CONFIG_HOME="${run_dir}/host/config" export XDG_DATA_HOME="${run_dir}/host/data" export XDG_STATE_HOME="${run_dir}/host/state" mkdir -p "${XDG_CONFIG_HOME}" "${XDG_DATA_HOME}" "${XDG_STATE_HOME}" gateway_name="openshell-e2e-${mode}-${host_port}" gateway_endpoint="http://127.0.0.1:${host_port}" export OPENSHELL_GATEWAY_ENDPOINT="${gateway_endpoint}" export OPENSHELL_GATEWAY="${gateway_name}" export OPENSHELL_BIN="${host_cli_bin}" if [ "${mode}" = host ]; then case "${gateway_driver}" in docker) e2e_align_docker_host_with_cli_context docker import \ --change 'ENTRYPOINT ["/openshell-sandbox"]' \ "${sandbox_runtime_archive}" \ "${sandbox_runtime_image}" >/dev/null docker import \ --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${supervisor_archive}" \ "${supervisor_image}" >/dev/null docker run --rm --network none "${supervisor_image}" --help >/dev/null ;; podman) podman import \ --change 'ENTRYPOINT ["/openshell-sandbox"]' \ "${sandbox_runtime_archive}" \ "${sandbox_runtime_image}" >/dev/null podman import \ --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${supervisor_archive}" \ "${supervisor_image}" >/dev/null podman run --rm --network none "${supervisor_image}" --help >/dev/null ;; esac runtime_log="${run_dir}/gateway.log" echo "==> Starting host gateway at ${gateway_endpoint}" start_child \ "${host_runtime_dir}" \ "${runtime_log}" \ "${host_gateway_bin}" \ --config "${gateway_config}" \ --bind-address 127.0.0.1 \ --port "${host_port}" \ --disable-tls else runtime_log="${run_dir}/vm.log" guest_launcher="${run_dir}/launch-gateway.sh" guest_launcher_path=/home/openshell/.cache/openshell-e2e/bin/launch-gateway guest_sandbox_runtime_archive_path=/home/openshell/.cache/openshell-e2e/sandbox-runtime.tar guest_supervisor_archive_path=/home/openshell/.cache/openshell-e2e/supervisor.tar config_payload="$(base64 <"${gateway_config}" | tr -d '\r\n')" jwt_signing_payload="$(base64 <"${jwt_source_dir}/signing.pem" | tr -d '\r\n')" jwt_public_payload="$(base64 <"${jwt_source_dir}/public.pem" | tr -d '\r\n')" jwt_kid_payload="$(base64 <"${jwt_source_dir}/kid" | tr -d '\r\n')" cat >"${guest_launcher}" < Timing: \${label}: \$((SECONDS - started_at))s" } phase_started_at=\${SECONDS} umask 077 state_root=/home/openshell/.cache/openshell-e2e config_path=\${state_root}/gateway.toml jwt_root=\${state_root}/gateway-jwt sudo chown -R "\$(id -u):\$(id -g)" /home/openshell/.cache chmod 0700 "\${state_root}" mkdir -p "\${state_root}/xdg/cache" "\${state_root}/xdg/config" "\${state_root}/xdg/data" "\${state_root}/xdg/state" "\${jwt_root}" printf '%s' '${config_payload}' | base64 --decode >"\${config_path}" printf '%s' '${jwt_signing_payload}' | base64 --decode >"\${jwt_root}/signing.pem" printf '%s' '${jwt_public_payload}' | base64 --decode >"\${jwt_root}/public.pem" printf '%s' '${jwt_kid_payload}' | base64 --decode >"\${jwt_root}/kid" chmod 0600 "\${config_path}" chmod 0600 "\${jwt_root}/signing.pem" "\${jwt_root}/public.pem" "\${jwt_root}/kid" export XDG_CONFIG_HOME=\${state_root}/xdg/config export XDG_CACHE_HOME=\${state_root}/xdg/cache export XDG_DATA_HOME=\${state_root}/xdg/data export XDG_STATE_HOME=\${state_root}/xdg/state report_timing "guest gateway setup" "\${phase_started_at}" phase_started_at=\${SECONDS} case '${gateway_driver}' in docker) docker import \ --change 'ENTRYPOINT ["/openshell-sandbox"]' \ "${guest_sandbox_runtime_archive_path}" \ "${sandbox_runtime_image}" >/dev/null docker import \ --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${guest_supervisor_archive_path}" \ "${supervisor_image}" >/dev/null docker run --rm --network none "${supervisor_image}" --help >/dev/null ;; podman) podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" import \ --change 'ENTRYPOINT ["/openshell-sandbox"]' \ "${guest_sandbox_runtime_archive_path}" \ "${sandbox_runtime_image}" >/dev/null podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" import \ --change 'ENTRYPOINT ["/openshell-supervisor"]' \ "${guest_supervisor_archive_path}" \ "${supervisor_image}" >/dev/null podman --url "unix:///run/user/\$(id -u)/podman/podman.sock" run \ --rm --network none "${supervisor_image}" --help >/dev/null ;; esac report_timing "${gateway_driver} supervisor import" "\${phase_started_at}" cd /home/openshell exec /usr/local/bin/openshell-gateway \ --config "\${config_path}" \ --bind-address 127.0.0.1 \ --port ${guest_port} \ --disable-tls EOF chmod 0700 "${guest_launcher}" vm_args=( nix run .#test-guest -- --distro "${vm}" ) for configuration in "${with_configurations[@]}"; do vm_args+=(--with "${configuration}") done vm_args+=( --copy "${guest_gateway_bin}:/usr/local/bin/openshell-gateway" --copy "${guest_launcher}:${guest_launcher_path}" --copy "${sandbox_runtime_archive}:${guest_sandbox_runtime_archive_path}" --copy "${supervisor_archive}:${guest_supervisor_archive_path}" --forward-port "${host_port}:${guest_port}" ) if [ "${keep}" -eq 1 ]; then vm_args+=(--keep) fi vm_args+=(-- "${guest_launcher_path}") echo "==> Starting ${vm} test guest gateway at ${gateway_endpoint}" start_child "${ROOT}" "${runtime_log}" "${vm_args[@]}" fi probe_gateway() { python3 - "${OPENSHELL_BIN}" "${1}" <<'PY' import os import subprocess import sys with open(sys.argv[2], "wb") as output: try: result = subprocess.run( [sys.argv[1], "status"], env={**os.environ, "NO_COLOR": "1"}, stdout=output, stderr=subprocess.STDOUT, timeout=5, check=False, ) except subprocess.TimeoutExpired: raise SystemExit(124) raise SystemExit(result.returncode) PY } wait_for_gateway() { local started_at=${SECONDS} local elapsed=0 local process_status local probe_log="${run_dir}/gateway-probe.log" local reported_timings=0 local timing_count report_vm_progress() { if [ "${mode}" != vm ]; then return fi timing_count="$(grep -c '^==> Timing:' "${runtime_log}" || true)" if [ "${timing_count}" -le "${reported_timings}" ]; then return fi sed -n 's/^==> Timing: / /p' "${runtime_log}" | sed -n "$((reported_timings + 1)),${timing_count}p" reported_timings=${timing_count} } echo "==> Waiting up to ${gateway_ready_timeout}s for gateway readiness" while :; do elapsed=$((SECONDS - started_at)) if [ "${elapsed}" -ge "${gateway_ready_timeout}" ]; then break fi if ! kill -0 "${child_pid}" 2>/dev/null; then if wait "${child_pid}"; then process_status=0 else process_status=$? fi child_pid= echo "ERROR: ${mode} gateway process exited before becoming ready" >&2 if [ "${process_status}" -eq 0 ]; then return 1 fi return "${process_status}" fi report_vm_progress if probe_gateway "${probe_log}" && grep -q "Connected" "${probe_log}"; then report_vm_progress echo "==> Gateway ready after ${elapsed}s" return 0 fi sleep 1 done echo "ERROR: gateway did not become ready within ${gateway_ready_timeout}s" >&2 if [ -s "${probe_log}" ]; then echo "=== last gateway probe ===" >&2 cat "${probe_log}" >&2 echo "=== end last gateway probe ===" >&2 fi return 1 } wait_for_gateway echo "==> Running E2E suite: ${suite_name}" cd "${ROOT}" cargo test \ --manifest-path e2e/rust/Cargo.toml \ --features e2e \ --test "${suite_name}" \ -- --nocapture