OpenShell/.github/workflows/e2e-test.yml

name: E2E Test

on:
  workflow_call:
    inputs:
      image-tag:
        description: "Image tag to test (typically the commit SHA)"
        required: true
        type: string
      runner:
        description: "GitHub Actions runner label"
        required: false
        type: string
        default: "linux-amd64-cpu8"
      checkout-ref:
        description: "Git ref to check out for test inputs (defaults to the workflow SHA)"
        required: false
        type: string
        default: ""
      cli-artifact-prefix:
        description: "Optional prebuilt CLI artifact prefix (artifact suffix is linux-<arch>)"
        required: false
        type: string
        default: ""
      gateway-artifact-prefix:
        description: "Optional prebuilt gateway artifact prefix (artifact suffix is linux-<arch>)"
        required: false
        type: string
        default: ""
      vm-driver-artifact-name:
        description: "Optional prebuilt VM driver artifact name"
        required: false
        type: string
        default: ""

permissions:
  actions: read
  contents: read
  packages: read

jobs:
  e2e:
    name: "E2E (${{ matrix.suite }})"
    runs-on: ${{ inputs.runner }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          - suite: python
            cmd: "mise run --no-deps --skip-deps e2e:python"
            apt_packages: ""
          - suite: oidc-python
            cmd: "mise run --no-deps --skip-deps e2e:oidc-python:docker"
            apt_packages: ""
          - suite: oidc-pkce-docker
            cmd: "mise run --no-deps --skip-deps e2e:oidc-pkce:docker"
            apt_packages: "openssh-client"
          - suite: rust-docker
            cmd: "mise run --no-deps --skip-deps e2e:rust"
            apt_packages: "openssh-client"
          - suite: mcp
            cmd: "mise run --no-deps --skip-deps e2e:mcp"
            apt_packages: ""
    container:
      image: ghcr.io/nvidia/openshell/ci:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
      options: --privileged
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock
        - /home/runner/_work:/home/runner/_work
    env:
      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      IMAGE_TAG: ${{ inputs.image-tag }}
      OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell
      OPENSHELL_REGISTRY_HOST: ghcr.io
      OPENSHELL_REGISTRY_NAMESPACE: nvidia/openshell
      OPENSHELL_REGISTRY_USERNAME: ${{ github.actor }}
      OPENSHELL_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ inputs['checkout-ref'] || github.sha }}
          persist-credentials: false

      - name: Use prebuilt OpenShell CLI
        if: inputs.cli-artifact-prefix != ''
        uses: ./.github/actions/setup-e2e-cli
        with:
          artifact-prefix: ${{ inputs.cli-artifact-prefix }}

      - name: Use prebuilt OpenShell gateway
        if: inputs.gateway-artifact-prefix != ''
        uses: ./.github/actions/setup-e2e-gateway
        with:
          artifact-prefix: ${{ inputs.gateway-artifact-prefix }}

      - name: Check out MCP conformance tests
        if: matrix.suite == 'mcp'
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: modelcontextprotocol/conformance
          # Pin after v0.1.16 to include the tools_call client scenario fix.
          ref: b9041ea41b0188581803459dbae71bc7e02fd995
          path: .cache/mcp-conformance
          persist-credentials: false

      - name: Install OS test dependencies
        if: matrix.apt_packages != ''
        env:
          APT_PACKAGES: ${{ matrix.apt_packages }}
        run: apt-get update && apt-get install -y ${APT_PACKAGES} && rm -rf /var/lib/apt/lists/*

      - name: Log in to GHCR with Docker
        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin

      - name: Install Python dependencies and generate protobuf stubs
        if: matrix.suite == 'python' || matrix.suite == 'oidc-python'
        run: uv sync --frozen && mise run --no-deps python:proto

      - name: Run tests
        env:
          OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }}
          OPENSHELL_MCP_CONFORMANCE_CLIENT_IMAGE: ${{ format('openshell-mcp-conformance-client:{0}', inputs.image-tag) }}
        run: ${{ matrix.cmd }}

  e2e-podman-rootless:
    name: E2E (rust-podman-rootless, ${{ matrix.runner }})
    # Run directly on the Ubuntu host so the test observes the host's AppArmor
    # and unprivileged-user-namespace policy. A privileged job container masks
    # the restrictions that production rootless Podman installations enforce.
    # Ubuntu 26.04 provides the supported Podman 5.x and pasta combination.
    # Re-add older/slirp4netns environments when direct callbacks through a
    # rootless-network namespace relay are supported.
    runs-on: ${{ matrix.runner }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          # Keep package versions explicit so hosted-runner tool overrides
          # cannot silently change the supported test environment.
          - runner: ubuntu-26.04
            podman_major: "5"
            podman_package_version: "5.7.0+ds2-3build1"
            conmon_package_version: "2.1.13+ds1-2"
    env:
      IMAGE_TAG: ${{ inputs.image-tag }}
      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      OPENSHELL_REGISTRY: ghcr.io/nvidia/openshell
      OPENSHELL_REGISTRY_HOST: ghcr.io
      OPENSHELL_REGISTRY_NAMESPACE: nvidia/openshell
      OPENSHELL_REGISTRY_USERNAME: ${{ github.actor }}
      OPENSHELL_REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
      OPENSHELL_SUPERVISOR_IMAGE: ${{ format('ghcr.io/nvidia/openshell/supervisor:{0}', inputs.image-tag) }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ inputs['checkout-ref'] || github.sha }}
          persist-credentials: false

      - name: Use prebuilt OpenShell CLI
        if: inputs.cli-artifact-prefix != ''
        uses: ./.github/actions/setup-e2e-cli
        with:
          artifact-prefix: ${{ inputs.cli-artifact-prefix }}

      - name: Use prebuilt OpenShell gateway
        if: inputs.gateway-artifact-prefix != ''
        uses: ./.github/actions/setup-e2e-gateway
        with:
          artifact-prefix: ${{ inputs.gateway-artifact-prefix }}

      - name: Install mise
        run: |
          curl https://mise.run | MISE_VERSION=v2026.4.25 sh
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
          echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH"

      - name: Install tools
        run: mise install --locked

      - name: Install Podman and build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential \
            clang \
            fuse-overlayfs \
            libssl-dev \
            libz3-dev \
            openssh-client \
            passt \
            pkg-config \
            "conmon=${{ matrix.conmon_package_version }}" \
            "podman=${{ matrix.podman_package_version }}" \
            uidmap
          # Hosted runners can place newer Podman and conmon binaries under
          # /usr/local ahead of Ubuntu's packages. Select the distro CLI and
          # use Podman's supported final config override for its conmon path.
          podman_config="${RUNNER_TEMP}/openshell-containers.conf"
          printf '%s\n' \
            '[engine]' \
            'conmon_path = ["/usr/bin/conmon"]' \
            > "${podman_config}"
          echo "/usr/bin" >> "${GITHUB_PATH}"
          echo "CONTAINERS_CONF_OVERRIDE=${podman_config}" >> "${GITHUB_ENV}"

      - name: Configure rootless Podman
        run: |
          set -euo pipefail
          if ! grep -q "^${USER}:" /etc/subuid; then
            sudo usermod --add-subuids 100000-165535 "$USER"
          fi
          if ! grep -q "^${USER}:" /etc/subgid; then
            sudo usermod --add-subgids 100000-165535 "$USER"
          fi
          runtime_dir="/run/user/$(id -u)"
          sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" "$runtime_dir"
          echo "XDG_RUNTIME_DIR=$runtime_dir" >> "$GITHUB_ENV"

      - name: Verify rootless Podman environment
        run: |
          set -euo pipefail
          podman_version="$(podman version --format '{{.Client.Version}}')"
          case "$podman_version" in
            "${{ matrix.podman_major }}".*) ;;
            *) echo "ERROR: expected Podman ${{ matrix.podman_major }}.x, found $podman_version" >&2; exit 1 ;;
          esac
          test "$(dpkg-query -W -f='${Version}' podman)" = "${{ matrix.podman_package_version }}"
          test "$(dpkg-query -W -f='${Version}' conmon)" = "${{ matrix.conmon_package_version }}"
          test "$(command -v podman)" = "/usr/bin/podman"
          test "$(podman info --format '{{.Host.Conmon.Path}}')" = "/usr/bin/conmon"
          test "$(podman info --format '{{.Host.Security.Rootless}}')" = "true"
          test "$(podman info --format '{{.Host.RootlessNetworkCmd}}')" = "pasta"
          test "$(sudo sysctl -n kernel.apparmor_restrict_unprivileged_userns)" = "1"
          echo "=== host ==="
          uname -a
          echo "=== AppArmor ==="
          cat /proc/self/attr/current
          sudo aa-status || true
          echo "=== Podman ==="
          podman version
          podman info --debug

      - name: Probe rootless capability bounding set
        run: |
          set -euo pipefail
          probe="$RUNNER_TEMP/openshell-capbset-probe"
          cc -static -O2 -Wall -Wextra -Werror \
            e2e/support/capbset-probe.c \
            -o "$probe"
          podman run --rm \
            --cap-add=SETPCAP \
            --volume "$probe:/openshell-capbset-probe:ro" \
            docker.io/library/alpine:3.22 \
            /openshell-capbset-probe

      - name: Log in to GHCR with Podman
        run: echo "${{ secrets.GITHUB_TOKEN }}" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin

      - name: Run rootless Podman E2E
        run: mise run --no-deps --skip-deps e2e:podman:rootless

      - name: Print AppArmor denials
        if: always()
        run: sudo dmesg | grep -E 'apparmor=.*DENIED|profile="unprivileged_userns"' | tail -100 || true

  e2e-vm:
    name: E2E (rust-vm)
    # libkrun needs KVM, so this job must run directly on a GitHub-hosted
    # Linux VM. GitHub-hosted macOS runners do not support nested
    # virtualization, and a job container would hide the host KVM device.
    runs-on: ubuntu-24.04
    timeout-minutes: 90
    env:
      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      MISE_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ inputs['checkout-ref'] || github.sha }}
          persist-credentials: false

      - name: Use prebuilt OpenShell CLI
        if: inputs.cli-artifact-prefix != ''
        uses: ./.github/actions/setup-e2e-cli
        with:
          artifact-prefix: ${{ inputs.cli-artifact-prefix }}

      - name: Use prebuilt OpenShell gateway
        if: inputs.gateway-artifact-prefix != ''
        uses: ./.github/actions/setup-e2e-gateway
        with:
          artifact-prefix: ${{ inputs.gateway-artifact-prefix }}

      - name: Use prebuilt OpenShell VM driver
        if: inputs.vm-driver-artifact-name != ''
        uses: ./.github/actions/setup-e2e-vm-driver
        with:
          artifact-name: ${{ inputs.vm-driver-artifact-name }}

      - name: Install system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential \
            clang \
            cmake \
            libclang-dev \
            libssl-dev \
            libz3-dev \
            openssh-client \
            pkg-config \
            socat \
            zstd

      - name: Enable KVM access
        run: |
          set -euo pipefail
          if [[ ! -c /dev/kvm ]]; then
            echo "::error::The GitHub-hosted runner did not expose /dev/kvm"
            lscpu
            grep -m1 -E '^(flags|Features)' /proc/cpuinfo || true
            ls -la /dev
            exit 1
          fi

          # Package installation can restart systemd-udevd, which reapplies
          # the default root:kvm 0660 mode. Install a persistent rule after
          # dependencies so later udev events preserve runner access.
          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
            | sudo tee /etc/udev/rules.d/99-kvm4all.rules
          sudo udevadm control --reload-rules
          sudo udevadm trigger --settle --name-match=kvm

          ls -l /dev/kvm
          exec 3<>/dev/kvm
          exec 3>&-

      - name: Validate VM host tools
        run: |
          command -v mke2fs
          command -v mkfs.ext4
          command -v debugfs

      - name: Install mise
        run: |
          curl https://mise.run | MISE_VERSION=v2026.4.25 sh
          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
          echo "$HOME/.local/share/mise/shims" >> "$GITHUB_PATH"

      - name: Install tools
        run: mise install --locked

      - name: Cache Rust target and registry
        uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
        with:
          shared-key: e2e-vm-linux-amd64
          cache-on-failure: "true"

      - name: Run VM E2E
        run: mise run --no-deps --skip-deps e2e:vm