ci(upgrade): wire Debian qualification into branch and releases

Signed-off-by: Evan Lezar <elezar@nvidia.com>

.agents/skills/test-release-canary/SKILL.md

@@ -39,9 +39,14 @@ lanes use the `rpm` installer on `fedora-podman-rootful` and
 gateway RPMs and matching runtime images. Branch RPM package builds run on
 every approved branch run, independently of optional E2E labels.
 
-The host-package jobs exercise fresh installs, not upgrades from a persisted
-schema-v1 gateway config. Validate Homebrew and RPM exact-default migration with
-the release-tooling and package lifecycle tests before relying on the canary.
+The host-package jobs exercise fresh installs, not upgrades from persisted
+gateway state. Release Dev and Release Tag additionally run a tmachine Ubuntu DEB
+upgrade lane: it installs the latest published release
+package and matching runtime images, upgrades to the candidate package, verifies
+an existing sandbox survives, and creates a new sandbox. That lane is part of
+the release conformance matrix, not this canary. Validate Homebrew exact-default
+migration with the release-tooling and package lifecycle tests before relying
+on the canary.
 
 The canary does not install or import `@nvidia/openshell-sdk`. TypeScript SDK
 validation lives in the `TypeScript SDK` branch check, including a publish

.github/workflows/branch-e2e.yml

@@ -24,6 +24,7 @@ jobs:
       run_integration: ${{ steps.labels.outputs.run_core_e2e }}
       run_core_e2e: ${{ steps.labels.outputs.run_core_e2e }}
       run_gpu_e2e: ${{ steps.labels.outputs.run_gpu_e2e }}
+      run_integration_upgrades: ${{ steps.labels.outputs.run_integration_upgrades }}
       run_kubernetes_ha_e2e: ${{ steps.labels.outputs.run_kubernetes_ha_e2e }}
       run_kubernetes_credential_drivers_e2e: ${{ steps.labels.outputs.run_kubernetes_credential_drivers_e2e }}
       run_any_e2e: ${{ steps.labels.outputs.run_any_e2e }}
@@ -45,17 +46,19 @@ jobs:
             push)
               run_core_e2e="$(jq -r 'index("test:e2e") != null' <<< "$LABELS_JSON")"
               run_gpu_e2e="$(jq -r 'index("test:e2e-gpu") != null' <<< "$LABELS_JSON")"
+              run_integration_upgrades="$(jq -r 'index("test:upgrade") != null' <<< "$LABELS_JSON")"
               run_kubernetes_ha_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
               run_kubernetes_credential_drivers_e2e="$(jq -r 'index("test:e2e-kubernetes") != null' <<< "$LABELS_JSON")"
               ;;
             *)
               run_core_e2e=true
               run_gpu_e2e=true
+              run_integration_upgrades=false
               run_kubernetes_ha_e2e=false
               run_kubernetes_credential_drivers_e2e=false
               ;;
           esac
-          if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ] || [ "$run_kubernetes_credential_drivers_e2e" = "true" ]; then
+          if [ "$run_core_e2e" = "true" ] || [ "$run_gpu_e2e" = "true" ] || [ "$run_integration_upgrades" = "true" ] || [ "$run_kubernetes_ha_e2e" = "true" ] || [ "$run_kubernetes_credential_drivers_e2e" = "true" ]; then
             run_any_e2e=true
           else
             run_any_e2e=false
@@ -64,6 +67,7 @@ jobs:
           {
             echo "run_core_e2e=$run_core_e2e"
             echo "run_gpu_e2e=$run_gpu_e2e"
+            echo "run_integration_upgrades=$run_integration_upgrades"
             echo "run_kubernetes_ha_e2e=$run_kubernetes_ha_e2e"
             echo "run_kubernetes_credential_drivers_e2e=$run_kubernetes_credential_drivers_e2e"
             echo "run_any_e2e=$run_any_e2e"
@@ -185,7 +189,7 @@ jobs:
 
   build-vm-driver:
     needs: [pr_metadata, version, build-binaries]
-    if: needs.pr_metadata.outputs.run_core_e2e == 'true'
+    if: needs.pr_metadata.outputs.run_core_e2e == 'true' || needs.pr_metadata.outputs.run_integration_upgrades == 'true'
     permissions:
       contents: read
     uses: ./.github/workflows/build-vm-driver.yml
@@ -205,7 +209,7 @@ jobs:
   build-deb:
     name: Build Debian packages
     needs: [pr_metadata, version, build-binaries, build-vm-driver]
-    if: needs.pr_metadata.outputs.run_integration == 'true'
+    if: needs.pr_metadata.outputs.run_integration == 'true' || needs.pr_metadata.outputs.run_integration_upgrades == 'true'
     permissions:
       contents: read
       packages: read
@@ -217,7 +221,7 @@ jobs:
   build-rpm:
     name: Build RPM packages
     needs: [pr_metadata, version, build-binaries]
-    if: needs.pr_metadata.outputs.run_integration == 'true'
+    if: needs.pr_metadata.outputs.run_integration == 'true' || needs.pr_metadata.outputs.run_integration_upgrades == 'true'
     permissions:
       actions: read
       contents: read
@@ -230,7 +234,7 @@ jobs:
 
   prepare-integration:
     needs: [pr_metadata, build-binaries, build-images, build-deb, build-rpm]
-    if: needs.pr_metadata.outputs.run_integration == 'true'
+    if: needs.pr_metadata.outputs.run_integration == 'true' || needs.pr_metadata.outputs.run_integration_upgrades == 'true'
     permissions:
       actions: read
       contents: read
@@ -239,10 +243,30 @@ jobs:
     with:
       deb-artifact-name: deb-linux-amd64
       rpm-artifact-name: rpm-linux-x86_64
+      include-deb-upgrade-source: ${{ needs.pr_metadata.outputs.run_integration_upgrades == 'true' }}
+
+  integration-upgrades:
+    name: Integration upgrades
+    needs: [pr_metadata, prepare-integration]
+    if: needs.pr_metadata.outputs.run_integration_upgrades == 'true'
+    permissions:
+      actions: read
+      contents: read
+      packages: read
+    uses: ./.github/workflows/integration-runner.yml
+    with:
+      category: upgrades
+      source-sha: ${{ needs.prepare-integration.outputs.source_sha }}
+      integration-inputs-artifact-id: ${{ needs.prepare-integration.outputs.integration_inputs_artifact_id }}
+      test-matrix: >-
+        [
+          {"environment":"ubuntu-docker-rootful","installer":"deb-upgrade-source","testsuite":"deb-upgrade"}
+        ]
 
   # Run driver-independent conformance tests.
   conformance-integration:
-    needs: prepare-integration
+    needs: [pr_metadata, prepare-integration]
+    if: needs.pr_metadata.outputs.run_integration == 'true'
     permissions:
       actions: read
       contents: read
@@ -262,7 +286,8 @@ jobs:
 
   # Run feature-specific integration tests:
   feature-specific-integration:
-    needs: prepare-integration
+    needs: [pr_metadata, prepare-integration]
+    if: needs.pr_metadata.outputs.run_integration == 'true'
     permissions:
       actions: read
       contents: read
@@ -282,7 +307,8 @@ jobs:
   # These suites require the binary installer's CLI path, local HTTP gateway,
   # and system service configuration for their Podman user-namespace fixtures.
   driver-specific-integration:
-    needs: prepare-integration
+    needs: [pr_metadata, prepare-integration]
+    if: needs.pr_metadata.outputs.run_integration == 'true'
     permissions:
       actions: read
       contents: read

.github/workflows/e2e-label-help.yml

@@ -19,7 +19,7 @@ permissions: {}
 jobs:
   hint:
     name: Post next-step hint for E2E label
-    if: github.event.label.name == 'test:e2e' || github.event.label.name == 'test:e2e-gpu' || github.event.label.name == 'test:e2e-kubernetes'
+    if: github.event.label.name == 'test:e2e' || github.event.label.name == 'test:e2e-gpu' || github.event.label.name == 'test:e2e-kubernetes' || github.event.label.name == 'test:upgrade'
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
@@ -50,6 +50,11 @@ jobs:
               build_summary="supervisor image"
               status_summary="The matching required CI gate status on this PR will flip green automatically once the run finishes."
               ;;
+            test:upgrade)
+              suite_summary="the Debian upgrade qualification"
+              build_summary="DEB and RPM packages, VM driver, sandbox image, and supervisor image"
+              status_summary="This is an optional proof-of-life suite; failures are visible in the workflow run but do not publish a required CI gate status."
+              ;;
             test:e2e-kubernetes)
               suite_summary="Kubernetes HA and credential-driver E2E"
               build_summary="gateway, sandbox, and supervisor images"

.github/workflows/release-dev.yml

@@ -111,6 +111,7 @@ jobs:
     with:
       deb-artifact-name: deb-linux-amd64
       rpm-artifact-name: rpm-linux-x86_64
+      include-deb-upgrade-source: true
 
   conformance-integration:
     needs: prepare-integration
@@ -126,6 +127,7 @@ jobs:
       test-matrix: >-
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
+          {"environment":"ubuntu-docker-rootful","installer":"deb-upgrade-source","testsuite":"deb-upgrade"},
           {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
           {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}

.github/workflows/release-tag.yml

@@ -162,6 +162,7 @@ jobs:
       source-sha: ${{ needs.compute-versions.outputs.source_sha }}
       deb-artifact-name: deb-linux-amd64
       rpm-artifact-name: rpm-linux-x86_64
+      include-deb-upgrade-source: true
 
   conformance-integration:
     needs: prepare-integration
@@ -177,6 +178,7 @@ jobs:
       test-matrix: >-
         [
           {"environment":"ubuntu-docker-rootful","installer":"deb","testsuite":"conformance"},
+          {"environment":"ubuntu-docker-rootful","installer":"deb-upgrade-source","testsuite":"deb-upgrade"},
           {"environment":"ubuntu-k3s","installer":"k3s","testsuite":"conformance"},
           {"environment":"fedora-podman-rootful","installer":"rpm","testsuite":"conformance"},
           {"environment":"fedora-podman-rootless","installer":"rpm","testsuite":"conformance"}

CI.md

@@ -126,11 +126,21 @@ retain the binary installer because their fixtures configure its system service,
 local HTTP gateway, and CLI path. The manual Integration Tests workflow defaults
 to the package installers and downloads the packages selected by its matrix.
 
-Three opt-in labels enable the long-running E2E suites:
+Four opt-in labels enable the long-running E2E suites:
+
+The `test:upgrade` label enables the Debian and RPM package builds, VM driver build,
+runtime-image preparation, and tmachine upgrade execution needed by that suite.
+It reuses the existing integration input-preparation job. Upgrade-only runs
+skip the regular integration suites; selecting both `test:e2e`
+and `test:upgrade` prepares inputs once for both sets of tests.
 
 - `test:e2e` runs the Docker, rootless Podman, Kubernetes, and VM E2E suites
   with both managed and standalone compute drivers in `Branch E2E Checks`
 - `test:e2e-gpu` runs GPU E2E in `Branch E2E Checks`
+- `test:upgrade` runs tmachine Debian upgrade qualification in
+  `Branch E2E Checks`: the latest published release packages and matching
+  runtime images are installed first, then the PR's packages are installed and
+  both existing and new sandboxes are verified
 - `test:e2e-kubernetes` runs Kubernetes E2E with the HA Helm overlay
   (`replicaCount: 2` and bundled PostgreSQL) and the credential-driver suite
   (Kubernetes Secrets plus Vault) in `Branch E2E Checks`
@@ -450,7 +460,7 @@ Flow:
 
 1. Open the PR. copy-pr-bot mirrors it to `pull-request/<N>` automatically.
 2. The mirror push runs `Branch Checks` automatically. `Required CI Gates` keeps the PR blocked until the mirror exists, matches the PR head SHA, and the required push-based workflow succeeds. The first `Branch E2E Checks` run only resolves metadata and skips expensive jobs unless an E2E label is already set.
-3. A maintainer applies `test:e2e`, `test:e2e-gpu`, and/or `test:e2e-kubernetes`. `E2E Label Help` posts a comment with a link to the existing gated workflow run.
+3. A maintainer applies `test:e2e`, `test:e2e-gpu`, `test:upgrade`, and/or `test:e2e-kubernetes`. `E2E Label Help` posts a comment with a link to the existing gated workflow run.
 4. The maintainer opens that link and clicks **Re-run all jobs**. This time `pr_metadata` sees the label and the build/E2E jobs run.
 5. When the run finishes, the matching `OpenShell / ...` gate status flips to green automatically.
 6. New commits push to the mirror automatically and re-trigger `Branch Checks` plus any labeled E2E jobs in `Branch E2E Checks`.
@@ -513,7 +523,7 @@ The bot's full administrator documentation is internal to NVIDIA. The only comma
 | File | Role |
 |---|---|
 | `.github/workflows/branch-checks.yml` | Required non-E2E checks. Triggers on `push: pull-request/[0-9]+` for PR mirrors and `merge_group` for queued merges. |
-| `.github/workflows/branch-e2e.yml` | Standard, GPU, Kubernetes HA, and Kubernetes credential-driver E2E. PR mirror pushes use `test:e2e`, `test:e2e-gpu`, and `test:e2e-kubernetes` labels; merge groups do not run E2E. |
+| `.github/workflows/branch-e2e.yml` | Standard, GPU, Debian upgrade, Kubernetes HA, and Kubernetes credential-driver E2E. PR mirror pushes use `test:e2e`, `test:e2e-gpu`, `test:upgrade`, and `test:e2e-kubernetes` labels; merge groups do not run E2E. |
 | `.github/workflows/build-binaries.yml`, `build-vm-driver.yml` | Shared binary matrices used by branch and release workflows. The VM driver remains separate because its build consumes the runtime binaries. |
 | `.github/workflows/build-images.yml` | Builds and pushes multi-platform images, then uploads the same OCI images as workflow artifacts. |
 | `.github/workflows/package-release-binaries.yml` | Packages raw build artifacts into release tarballs without rebuilding them. |