test(tmachine): add K3s conformance scenario (#3848)

* test(tmachine): add K3s conformance scenario Signed-off-by: Simon Scatton <sscatton@nvidia.com> * refactor(tmachine): use Helm values file for K3s installer Signed-off-by: Simon Scatton <sscatton@nvidia.com> * ci(tmachine): run K3s conformance in integration jobs Signed-off-by: Simon Scatton <sscatton@nvidia.com> * ci(tmachine): verify installer scripts and document version baseline Signed-off-by: Simon Scatton <sscatton@nvidia.com> --------- Signed-off-by: Simon Scatton <sscatton@nvidia.com>
2026-10-02 07:34:45 +08:00 · 2026-09-30 14:49:09 +00:00
parent 5acaaba192
commit 21fea95935
8 changed files with 285 additions and 1 deletions
@@ -224,6 +224,7 @@ jobs:
      test-matrix: >-
        [
          {"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"conformance"},
+          {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
          {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"},
          {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"}
        ]
@@ -25,6 +25,7 @@ on:
        default: >-
          [
            {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
+            {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
            {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"},
            {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"}
          ]
@@ -67,6 +68,7 @@ jobs:
      - uses: ./.github/actions/setup-nix

      - name: Cache tmachine disks
+        if: matrix.environment != 'ubuntu-k3s'
        uses: actions/cache@caa296126883cff596d87d8935842f9db880ef25 # v5.1.0
        with:
          path: ~/.cache/tmachine
@@ -23,6 +23,7 @@ on:
        default: >-
          [
            {"environment":"ubuntu-docker-rootful","installer":"binaries","testsuite":"conformance"},
+            {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
            {"environment":"fedora-podman-rootful","installer":"binaries","testsuite":"conformance"},
            {"environment":"fedora-podman-rootless","installer":"binaries","testsuite":"conformance"}
          ]
@@ -89,12 +89,16 @@ jobs:
      - name: Log in to GHCR
        run: echo "${{ github.token }}" | docker login ghcr.io -u "${GITHUB_ACTOR}" --password-stdin

-      - name: Export runtime images
+      - name: Export OpenShell images
        env:
          IMAGE_TAG: ${{ steps.artifact-run.outputs.source_sha }}
        run: |
          mkdir -p artifacts/images

+          docker pull "ghcr.io/nvidia/openshell/gateway:${IMAGE_TAG}"
+          docker tag "ghcr.io/nvidia/openshell/gateway:${IMAGE_TAG}" openshell/gateway:tmachine
+          docker save --output artifacts/images/openshell-gateway-tmachine.tar openshell/gateway:tmachine
+
          docker pull "ghcr.io/nvidia/openshell/sandbox:${IMAGE_TAG}"
          docker tag "ghcr.io/nvidia/openshell/sandbox:${IMAGE_TAG}" openshell/sandbox:tmachine
          docker save --output artifacts/images/openshell-sandbox-tmachine.tar openshell/sandbox:tmachine
@@ -109,6 +113,9 @@ jobs:
      - name: Build test workload images
        run: nix run .#build-artifacts-test-images

+      - name: Package Helm chart
+        run: nix run .#build-artifacts-helm
+
      - name: Upload integration inputs
        id: upload-integration-inputs
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -39,6 +39,13 @@ The GitHub ruleset should require the `OpenShell / ...` statuses published by
 `Required CI Gates` plus the direct `OpenShell / Trivy Changes` result, not the
 push-triggered workflow jobs themselves.

+### K3s conformance version baseline
+
+The tmachine `ubuntu-k3s` conformance lane pins Agent Sandbox v0.5.0 as the
+compatibility baseline for the v1beta1 Sandbox API. It does not track the local
+K3s development default, currently v1.0.3. OpenShell also supports v0.4.6 through
+its v1alpha1 fallback, so v0.5.0 is not the overall minimum supported version.
+
 ### Run only the policy advisor conformance tests

 Manually dispatch `Integration Tests` on the candidate branch with an
@@ -0,0 +1,67 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Install K3s and Helm
+  hosts: all
+  become: true
+  gather_facts: false
+  vars:
+    helm_version: 4.2.0
+    k3s_version: 1.36.3+k3s1
+  tasks:
+    - name: Wait for SSH
+      ansible.builtin.wait_for_connection:
+
+    - name: Install download prerequisites
+      ansible.builtin.apt:
+        name:
+          - ca-certificates
+          - curl
+        state: present
+        update_cache: true
+
+    - name: Download K3s installer
+      ansible.builtin.get_url:
+        url: https://get.k3s.io
+        checksum: sha256:e5cc3b3d9dfc1662c2d9be6da5abc9a4cd317d6abc3a5ffc02e3dd3248207fee
+        dest: /tmp/install-k3s.sh
+        mode: "0755"
+
+    - name: Install K3s
+      ansible.builtin.command:
+        argv:
+          - /tmp/install-k3s.sh
+          - server
+          - --disable=servicelb
+          - --disable=traefik
+        creates: /usr/local/bin/k3s
+      environment:
+        INSTALL_K3S_VERSION: "v{{ k3s_version }}"
+
+    - name: Wait for K3s
+      ansible.builtin.command:
+        argv:
+          - /usr/local/bin/k3s
+          - kubectl
+          - wait
+          - --for=condition=Ready
+          - node
+          - --all
+          - --timeout=120s
+      changed_when: false
+
+    - name: Download Helm installer
+      ansible.builtin.get_url:
+        url: https://raw.githubusercontent.com/helm/helm/06468084e85c244c712834933d25ea232a4c2093/scripts/get-helm-4
+        checksum: sha256:b68c5f694cff19f14ee8a5784ffd3de27fa7034ec8f973d703fc6fb85496ced7
+        dest: /tmp/install-helm.sh
+        mode: "0755"
+
+    - name: Install Helm
+      ansible.builtin.command:
+        argv:
+          - /tmp/install-helm.sh
+        creates: /usr/local/bin/helm
+      environment:
+        DESIRED_VERSION: "v{{ helm_version }}"
@@ -0,0 +1,175 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+---
+- name: Install OpenShell on K3s
+  hosts: all
+  become: true
+  gather_facts: false
+  tasks:
+    - name: Wait for SSH
+      ansible.builtin.wait_for_connection:
+
+    - name: Install OpenShell CLI
+      ansible.builtin.copy:
+        src: "{{ openshell_cli_binary }}"
+        dest: /usr/local/bin/openshell
+        mode: "0755"
+
+    - name: Create OpenShell artifact directory
+      ansible.builtin.file:
+        path: /var/lib/openshell/artifacts
+        state: directory
+        mode: "0700"
+
+    - name: Copy OpenShell images
+      ansible.builtin.copy:
+        src: "{{ item.src }}"
+        dest: "/var/lib/openshell/artifacts/{{ item.name }}.tar"
+        mode: "0600"
+      loop:
+        - name: gateway
+          src: "{{ openshell_gateway_image }}"
+        - name: sandbox
+          src: "{{ openshell_sandbox_image }}"
+        - name: supervisor
+          src: "{{ openshell_supervisor_image }}"
+
+    - name: Import OpenShell images into K3s
+      ansible.builtin.command:
+        argv:
+          - /usr/local/bin/k3s
+          - ctr
+          - --namespace
+          - k8s.io
+          - images
+          - import
+          - "/var/lib/openshell/artifacts/{{ item }}.tar"
+      loop:
+        - gateway
+        - sandbox
+        - supervisor
+
+    - name: Copy OpenShell Helm chart
+      ansible.builtin.copy:
+        src: "{{ openshell_helm_chart }}"
+        dest: /var/lib/openshell/artifacts/helm-chart.tgz
+        mode: "0600"
+
+    - name: Write OpenShell Helm values
+      ansible.builtin.copy:
+        dest: /var/lib/openshell/artifacts/values.yaml
+        mode: "0600"
+        content: |
+          global:
+            image:
+              registry: ""
+          gateway:
+            image:
+              repository: openshell/gateway
+              tag: tmachine
+              pullPolicy: Never
+          sandboxRuntime:
+            image:
+              repository: openshell/sandbox
+              tag: tmachine
+              pullPolicy: never
+          supervisor:
+            image:
+              repository: openshell/supervisor
+              tag: tmachine
+              pullPolicy: never
+          networkPolicy:
+            enabled: true
+          server:
+            auth:
+              allowUnauthenticatedUsers: true
+            disableTls: true
+            telemetryEnabled: false
+
+    - name: Install Agent Sandbox
+      ansible.builtin.command:
+        argv:
+          - /usr/local/bin/k3s
+          - kubectl
+          - apply
+          - --filename
+          - "https://github.com/kubernetes-sigs/agent-sandbox/releases/download/v{{ agent_sandbox_version }}/manifest.yaml"
+
+    - name: Wait for Agent Sandbox CRD
+      ansible.builtin.command:
+        argv:
+          - /usr/local/bin/k3s
+          - kubectl
+          - wait
+          - --for=condition=Established
+          - crd/sandboxes.agents.x-k8s.io
+          - --timeout=120s
+      changed_when: false
+
+    - name: Wait for Agent Sandbox controller
+      ansible.builtin.command:
+        argv:
+          - /usr/local/bin/k3s
+          - kubectl
+          - --namespace
+          - agent-sandbox-system
+          - rollout
+          - status
+          - deployment/agent-sandbox-controller
+          - --timeout=300s
+      changed_when: false
+
+    - name: Install OpenShell Helm chart
+      ansible.builtin.command:
+        argv:
+          - /usr/local/bin/helm
+          - install
+          - openshell
+          - /var/lib/openshell/artifacts/helm-chart.tgz
+          - --namespace
+          - openshell
+          - --create-namespace
+          - --values
+          - /var/lib/openshell/artifacts/values.yaml
+          - --wait
+          - --timeout=5m
+      environment:
+        KUBECONFIG: /etc/rancher/k3s/k3s.yaml
+
+    - name: Install gateway port-forward service
+      ansible.builtin.copy:
+        dest: /etc/systemd/system/openshell-k3s-port-forward.service
+        mode: "0644"
+        content: |
+          [Unit]
+          Description=OpenShell K3s gateway port forward
+          After=k3s.service
+          Requires=k3s.service
+
+          [Service]
+          ExecStart=/usr/local/bin/k3s kubectl --namespace openshell port-forward --address 127.0.0.1 service/openshell 17670:8080
+          Restart=always
+          RestartSec=1
+
+          [Install]
+          WantedBy=multi-user.target
+
+    - name: Start gateway port-forward service
+      ansible.builtin.systemd_service:
+        name: openshell-k3s-port-forward.service
+        daemon_reload: true
+        enabled: true
+        state: started
+
+    - name: Wait for OpenShell gateway
+      ansible.builtin.wait_for:
+        host: 127.0.0.1
+        port: 17670
+        timeout: 60
+
+- name: Register OpenShell gateway for test client
+  hosts: all
+  gather_facts: false
+  roles:
+    - openshell_client
@@ -43,6 +43,17 @@ let
          ];
        };
      }
+      {
+        name = "ubuntu-k3s";
+        machine = "ubuntu";
+        setup = {
+          use_galaxy = false;
+          playbooks = [
+            "ansible/playbooks/nextest.yaml"
+            "ansible/playbooks/k3s.yaml"
+          ];
+        };
+      }
      {
        name = "fedora-podman-rootful";
        machine = "fedora";
@@ -70,6 +81,19 @@ let
    ];

    installers = [
+      {
+        name = "k3s";
+        use_galaxy = false;
+        playbooks = [ "ansible/playbooks/openshell-k3s.yaml" ];
+        inputs = {
+          agent_sandbox_version = "0.5.0";
+          openshell_cli_binary = "../artifacts/binaries/${muslTarget}/openshell";
+          openshell_gateway_image = "../artifacts/images/openshell-gateway-tmachine.tar";
+          openshell_helm_chart = "../artifacts/helm/helm-chart-0.0.0.tgz";
+          openshell_sandbox_image = "../artifacts/images/openshell-sandbox-tmachine.tar";
+          openshell_supervisor_image = "../artifacts/images/openshell-supervisor-tmachine.tar";
+        };
+      }
      {
        name = "none";
        use_galaxy = false;