name: Release Canary

on:
  workflow_dispatch:
  workflow_run:
    workflows: ["Release Dev"]
    types: [completed]

permissions:
  actions: read
  contents: read

defaults:
  run:
    shell: bash

env:
  OPENSHELL_VERSION: dev
  OPENSHELL_TELEMETRY_ENABLED: "false"

jobs:
  macos:
    name: macOS Homebrew
    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
    runs-on: macos-latest-xlarge
    timeout-minutes: 20
    steps:
      - name: Ensure VM driver
        run: |
          launchctl setenv OPENSHELL_COMPUTE_DRIVER vm
          launchctl setenv OPENSHELL_TELEMETRY_ENABLED "$OPENSHELL_TELEMETRY_ENABLED"

      - name: Install and check gateway status
        run: |
          set -euo pipefail
          curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh
          openshell --version
          openshell status

      # GitHub-hosted macOS runners do not expose the Hypervisor.framework
      # support libkrun needs. Sandbox launch is covered by the VM E2E lane.
      - name: Collect Homebrew diagnostics
        if: failure()
        run: |
          set +e
          brew services info openshell
          for log in \
            "$(brew --prefix)/var/log/openshell/openshell-gateway.out.log" \
            "$(brew --prefix)/var/log/openshell/openshell-gateway.err.log"; do
            if [ -f "$log" ]; then
              echo "--- $log ---"
              tail -n 300 "$log"
            fi
          done

  ubuntu-deb:
    name: Ubuntu DEB
    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
      - name: Remove snapd
        run: |
          set -euo pipefail
          snaps=$(snap list 2>/dev/null | awk 'NR > 1 { print $1 }')
          if [ -n "$snaps" ]; then
            sudo snap remove $snaps
          fi
          sudo apt-get -y --purge remove snapd
          ! command -v snap >/dev/null 2>&1

      - name: Ensure Docker
        run: |
          if ! command -v docker >/dev/null 2>&1; then
            sudo apt-get update
            sudo apt-get install -y docker.io
          fi
          sudo systemctl start docker || sudo service docker start
          mkdir -p "${HOME}/.config/openshell"
          printf 'OPENSHELL_COMPUTE_DRIVER=docker\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \
            "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env"
          docker info

      - name: Install and check status
        run: |
          set -euo pipefail
          curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh
          openshell status
          sandbox="rc-${GITHUB_RUN_ID}"
          openshell sandbox create --name "$sandbox" --detach
          openshell sandbox exec --name "$sandbox" --no-tty -- true
          openshell sandbox delete "$sandbox"

  fedora:
    name: Fedora RPM
    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
    runs-on: linux-amd64-cpu8
    timeout-minutes: 20
    env:
      FEDORA_CANARY_CONTAINER: openshell-fedora-canary-${{ github.run_id }}-${{ github.run_attempt }}
    steps:
      - name: Start Fedora systemd container and root user manager
        run: |
          set -euo pipefail

          docker run --detach \
            --name "${FEDORA_CANARY_CONTAINER}" \
            --privileged \
            --cgroupns=host \
            --tmpfs /run \
            --tmpfs /tmp \
            --volume /sys/fs/cgroup:/sys/fs/cgroup:rw \
            fedora:latest \
            bash -lc 'dnf install -y curl dbus-daemon podman systemd && exec /usr/sbin/init'

          for _ in $(seq 1 120); do
            if docker exec "${FEDORA_CANARY_CONTAINER}" systemctl list-units --no-pager >/dev/null 2>&1; then
              break
            fi
            if [ "$(docker inspect -f '{{.State.Running}}' "${FEDORA_CANARY_CONTAINER}")" != "true" ]; then
              echo "::error::Fedora systemd container exited before systemd became reachable"
              docker logs "${FEDORA_CANARY_CONTAINER}" >&2 || true
              exit 1
            fi
            sleep 1
          done

          if ! docker exec "${FEDORA_CANARY_CONTAINER}" systemctl list-units --no-pager >/dev/null 2>&1; then
            echo "::error::Fedora systemd container did not become reachable within 120s"
            docker logs "${FEDORA_CANARY_CONTAINER}" >&2 || true
            exit 1
          fi

          docker exec --interactive "${FEDORA_CANARY_CONTAINER}" env \
            HOME=/root \
            XDG_RUNTIME_DIR=/run/user/0 \
            DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \
            bash -s <<'EOF'
          set -euo pipefail
          # install.sh manages the RPM gateway as a systemd user unit. This
          # container is booted with systemd as PID 1, but it still has no
          # login session. Start root's user manager explicitly so the
          # installer can test service restart and gateway registration
          # instead of its "restart later" fallback.
          mkdir -p "${XDG_RUNTIME_DIR}"
          chmod 700 "${XDG_RUNTIME_DIR}"
          systemctl start user-runtime-dir@0.service || true
          systemctl start user@0.service

          for _ in $(seq 1 30); do
            if systemctl --user daemon-reload; then
              break
            fi
            sleep 1
          done
          if ! systemctl --user daemon-reload; then
            systemctl status user@0.service --no-pager >&2 || true
            journalctl -u user@0.service --no-pager -n 80 >&2 || true
            systemctl --user status --no-pager >&2 || true
            exit 1
          fi
          EOF

      - name: Install and check status
        run: |
          set -euo pipefail

          docker exec --interactive "${FEDORA_CANARY_CONTAINER}" env \
            HOME=/root \
            XDG_RUNTIME_DIR=/run/user/0 \
            DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/0/bus \
            OPENSHELL_VERSION="$OPENSHELL_VERSION" \
            OPENSHELL_TELEMETRY_ENABLED="$OPENSHELL_TELEMETRY_ENABLED" \
            CANARY_SANDBOX="rc-${GITHUB_RUN_ID}" \
            INSTALL_SH_URL="https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh" \
            bash -s <<'EOF'
          set -euo pipefail
          mkdir -p "${HOME}/.config/openshell"
          printf 'OPENSHELL_COMPUTE_DRIVER=podman\nOPENSHELL_TELEMETRY_ENABLED=%s\n' \
            "$OPENSHELL_TELEMETRY_ENABLED" > "${HOME}/.config/openshell/gateway.env"
          podman info
          curl -LsSf "${INSTALL_SH_URL}" | sh
          openshell status
          openshell sandbox create --name "$CANARY_SANDBOX" --detach
          openshell sandbox exec --name "$CANARY_SANDBOX" --no-tty -- true
          openshell sandbox delete "$CANARY_SANDBOX"
          EOF

      - name: Stop Fedora systemd container
        if: always()
        run: |
          docker rm -f "${FEDORA_CANARY_CONTAINER}" >/dev/null 2>&1 || true

  ubuntu-snap-system-docker:
    name: Ubuntu snap with system Docker
    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
    runs-on: ubuntu-latest
    env:
      OPENSHELL_INSTALL_METHOD: snap
    timeout-minutes: 20
    steps:
      - name: Install snapd
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y snapd
          sudo systemctl enable --now snapd.socket
          sudo systemctl start snapd
          sudo snap wait system seed.loaded

      - name: Ensure Docker
        run: |
          set -euo pipefail
          if ! command -v docker >/dev/null 2>&1; then
            sudo apt-get update
            sudo apt-get install -y docker.io
          fi
          sudo systemctl start docker || sudo service docker start
          docker info

      - name: Install and check status
        run: |
          set -euo pipefail
          systemctl --user set-environment \
            "OPENSHELL_TELEMETRY_ENABLED=${OPENSHELL_TELEMETRY_ENABLED}"
          curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh |
            DOCKER_HOST=unix:///tmp/nonexistent-docker.sock DOCKER_CONTEXT=nonexistent sh
          sudo snap list openshell
          if sudo snap list docker >/dev/null 2>&1; then
            echo "install.sh unexpectedly installed the Docker snap" >&2
            exit 1
          fi
          snap info openshell | grep -E '^tracking: +latest/edge$'
          docker info
          sudo snap connections openshell | grep -E '^docker +openshell:docker +:docker +'
          openshell --version
          openshell.prover --version
          snap services openshell
          systemctl --user is-enabled --quiet snap.openshell.user-gateway.service
          systemctl --user is-active snap.openshell.user-gateway.service
          ! systemctl is-enabled --quiet snap.openshell.system-gateway.service
          ! systemctl is-active --quiet snap.openshell.system-gateway.service
          journalctl --user -u snap.openshell.user-gateway.service --no-pager |
            grep -F "mTLS user authentication enabled"
          openshell gateway list | grep -F "https://127.0.0.1:17670"
          openshell status

      - name: Check a policy boundary with the snap prover
        run: |
          set -euo pipefail
          prover_dir=$(mktemp -d "$HOME/openshell-prover-canary.XXXXXX")
          trap 'rm -rf "$prover_dir"' EXIT
          cat >"$prover_dir/boundary.yaml" <<'EOF'
          version: 1
          filesystem_policy:
            read_only:
              - /usr
              - /etc
          EOF
          cat >"$prover_dir/candidate.yaml" <<'EOF'
          version: 1
          filesystem_policy:
            read_only:
              - /usr
          EOF
          result=$(openshell.prover check "$prover_dir/candidate.yaml" \
            --boundary "$prover_dir/boundary.yaml")
          grep -q '^result: within_boundary$' <<<"$result"

      - name: Create and exercise a sandbox
        run: |
          set -euo pipefail
          sandbox="rc-${GITHUB_RUN_ID}"
          openshell sandbox create --name "$sandbox" --detach
          openshell sandbox exec --name "$sandbox" --no-tty -- true
          openshell sandbox delete "$sandbox"

      - name: Collect snap diagnostics
        if: failure()
        run: |
          set +e
          docker info
          sudo systemctl status docker.service --no-pager
          sudo journalctl -b -u docker.service --no-pager -n 300
          sudo snap services openshell
          sudo snap connections openshell
          sudo snap changes
          systemctl --user status snap.openshell.user-gateway.service --no-pager
          journalctl --user -u snap.openshell.user-gateway.service --no-pager -n 300
          sudo systemctl status snap.openshell.system-gateway.service --no-pager
          sudo journalctl -b -u snap.openshell.system-gateway.service --no-pager -n 300
          sudo journalctl -b -u snapd.service --no-pager -n 300
          snap logs openshell.user-gateway -n=300
          sudo ss -ltnp '( sport = :17670 )'

  ubuntu-snap-docker-preflight:
    name: Ubuntu snap Docker preflight
    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
    runs-on: ubuntu-latest
    env:
      OPENSHELL_INSTALL_METHOD: snap
    timeout-minutes: 20
    steps:
      - name: Install snapd
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y snapd
          sudo systemctl enable --now snapd.socket
          sudo systemctl start snapd
          sudo snap wait system seed.loaded

      - name: Remove system Docker
        run: |
          set -euo pipefail
          sudo systemctl stop docker.service docker.socket 2>/dev/null || true
          sudo apt-get -y --purge remove docker-ce docker-ce-cli containerd.io \
            docker-buildx-plugin docker-compose-plugin docker.io 2>/dev/null || true
          ! command -v docker >/dev/null 2>&1

      - name: Verify installer rejects missing Docker
        run: |
          set -euo pipefail
          curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh -o install.sh
          if OPENSHELL_VERSION=dev sh install.sh >install.log 2>&1; then
            echo "install.sh unexpectedly succeeded without Docker" >&2
            cat install.log >&2
            exit 1
          fi
          cat install.log
          grep -F "Docker is required before installing the OpenShell snap" install.log
          ! sudo snap list docker >/dev/null 2>&1
          ! sudo snap list openshell >/dev/null 2>&1

      - name: Verify installer rejects Docker snap
        run: |
          set -euo pipefail
          sudo snap install docker
          if OPENSHELL_VERSION=dev sh install.sh >docker-snap.log 2>&1; then
            echo "install.sh unexpectedly succeeded with the Docker snap" >&2
            cat docker-snap.log >&2
            exit 1
          fi
          cat docker-snap.log
          grep -F "the Docker snap is not currently compatible with OpenShell" docker-snap.log
          sudo snap list docker >/dev/null
          ! sudo snap list openshell >/dev/null 2>&1

      - name: Collect snap diagnostics
        if: failure()
        run: |
          set +e
          cat install.log
          cat docker-snap.log
          sudo snap changes
          sudo journalctl -b -u snapd.service --no-pager -n 300

  kubernetes:
    name: Kubernetes Helm (kind)
    if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
    runs-on: ubuntu-latest
    timeout-minutes: 20
    env:
      KIND_CLUSTER_NAME: release-canary-${{ github.run_id }}
      RELEASE_NAME: openshell
      RELEASE_NAMESPACE: openshell
      KIND_GATEWAY_NAME: kind
      AGENT_SANDBOX_VERSION: v1.0.3
    steps:
      - name: Checkout Agent Sandbox helper
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          sparse-checkout: |
            e2e/support/install-agent-sandbox.sh
          sparse-checkout-cone-mode: false
          persist-credentials: false

      - name: Install Helm
        uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1

      - name: Create kind cluster
        uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # v1.14.0
        with:
          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
          wait: 120s

      - name: Install Agent Sandbox controller
        run: bash e2e/support/install-agent-sandbox.sh

      - name: Install OpenShell Helm chart from GHCR OCI
        run: |
          set -euo pipefail
          helm install "$RELEASE_NAME" oci://ghcr.io/nvidia/openshell/helm-chart \
            --version 0.0.0-dev \
            --namespace "$RELEASE_NAMESPACE" --create-namespace \
            --set server.disableTls=true \
            --set server.auth.allowUnauthenticatedUsers=true \
            --set "server.telemetryEnabled=${OPENSHELL_TELEMETRY_ENABLED}" \
            --wait --timeout 5m

      - name: Verify gateway pod is Ready
        run: |
          set -euo pipefail
          kubectl wait --namespace "$RELEASE_NAMESPACE" \
            --for=condition=Ready pod \
            --selector="app.kubernetes.io/name=openshell,app.kubernetes.io/instance=${RELEASE_NAME}" \
            --timeout=300s

      - name: Port-forward gateway service
        run: |
          set -euo pipefail
          nohup kubectl port-forward --namespace "$RELEASE_NAMESPACE" \
            "svc/${RELEASE_NAME}" 8080:8080 \
            > port-forward.log 2>&1 &
          echo $! > port-forward.pid
          for _ in $(seq 1 30); do
            if (echo > /dev/tcp/127.0.0.1/8080) >/dev/null 2>&1; then
              echo "port-forward is reachable"
              exit 0
            fi
            sleep 1
          done
          echo "port-forward did not become reachable" >&2
          cat port-forward.log >&2
          exit 1

      - name: Remove snapd
        run: |
          set -euo pipefail
          snaps=$(snap list 2>/dev/null | awk 'NR > 1 { print $1 }')
          if [ -n "$snaps" ]; then
            sudo snap remove $snaps
          fi
          sudo apt-get -y --purge remove snapd
          ! command -v snap >/dev/null 2>&1

      - name: Install OpenShell CLI
        run: |
          set -euo pipefail
          mkdir -p "${HOME}/.config/openshell"
          printf 'OPENSHELL_COMPUTE_DRIVER=docker\n' > "${HOME}/.config/openshell/gateway.env"
          curl -LsSf https://raw.githubusercontent.com/NVIDIA/OpenShell/${{ github.event.workflow_run.head_sha || github.sha }}/install.sh | sh

      - name: Register kind gateway and check status
        run: |
          set -euo pipefail
          openshell gateway add http://127.0.0.1:8080 --local --name "$KIND_GATEWAY_NAME"
          openshell status
          sandbox="rc-${GITHUB_RUN_ID}"
          openshell sandbox create --name "$sandbox" --detach
          openshell sandbox exec --name "$sandbox" --no-tty -- true
          openshell sandbox delete "$sandbox"

      - name: Diagnostics on failure
        if: failure()
        run: |
          set +e
          echo "--- helm status ---"
          helm status "$RELEASE_NAME" --namespace "$RELEASE_NAMESPACE"
          echo "--- helm get manifest ---"
          helm get manifest "$RELEASE_NAME" --namespace "$RELEASE_NAMESPACE"
          echo "--- get all ---"
          kubectl get all --namespace "$RELEASE_NAMESPACE"
          echo "--- describe pods ---"
          kubectl describe pods --namespace "$RELEASE_NAMESPACE"
          echo "--- pod logs ---"
          kubectl logs --namespace "$RELEASE_NAMESPACE" \
            --selector="app.kubernetes.io/name=openshell,app.kubernetes.io/instance=${RELEASE_NAME}" \
            --tail=200 --all-containers --prefix
          echo "--- port-forward log ---"
          cat port-forward.log 2>/dev/null
          echo "--- openshell gateway list ---"
          openshell gateway list 2>/dev/null
          echo "--- openshell version ---"
          openshell --version 2>/dev/null