feat(testing): support independent gateway and supervisor image overrides (#3341)

* feat(testing): normalize configurable test images

Signed-off-by: Bobbins228 <mcampbel@redhat.com>
Signed-off-by: Kris Hicks <khicks@nvidia.com>

* feat(helm): add global image overrides

Signed-off-by: Bobbins228 <mcampbel@redhat.com>

* feat(helm): support image registry overrides

Signed-off-by: Bobbins228 <mcampbel@redhat.com>
Signed-off-by: Kris Hicks <khicks@nvidia.com>

* refactor(helm): simplify image configuration

Signed-off-by: Bobbins228 <mcampbel@redhat.com>

* fix(e2e): avoid reloading reused kind sandbox image

Signed-off-by: Bobbins228 <mcampbel@redhat.com>

* fix(helm): default sandbox image to nvcr.io/nvidia/base/ubuntu:24.04

Signed-off-by: Kris Hicks <khicks@nvidia.com>

---------

Signed-off-by: Bobbins228 <mcampbel@redhat.com>
Signed-off-by: Kris Hicks <khicks@nvidia.com>
Co-authored-by: Kris Hicks <khicks@nvidia.com>
This commit is contained in:
Mark Campbell
2026-09-23 23:35:30 +00:00
committed by GitHub
co-authored by Kris Hicks
parent 490055b426
commit 679b190677
22 changed files with 641 additions and 156 deletions
+3 -2
View File
@@ -38,8 +38,9 @@ on local Docker Desktop, or via `--add-host ...:host-gateway` on local Linux.
The generated policy uses `protocol: mcp`, inserts the conformance runner's spec revision into the endpoint allowlist, and sets `mcp.allow_all_known_mcp_methods: true` so omitted rule methods use the endpoint MCP method profile. OpenShell enforces that allowlist on each non-initialize request using `MCP-Protocol-Version`, with `2025-03-26` as the missing-header fallback. The conformance runner selects the revision used by its client and server; OpenShell's request-version check does not yet provide complete revision-specific message parsing or response validation. The policy keeps OpenShell deny-by-default at the network boundary while allowing the upstream scenarios to exercise MCP behavior. The policy body lives in `policy-template.yaml`; the wrapper renders its MCP revision, host, port, and path placeholders from the upstream server URL.
For local runs, the wrapper builds `openshell/supervisor:dev` automatically
when no supervisor image override is set. Set `OPENSHELL_DOCKER_SUPERVISOR_IMAGE`
or `OPENSHELL_SUPERVISOR_IMAGE` to use a prebuilt pullable image instead.
when no supervisor image override is set. Set `SUPERVISOR_IMAGE` to use a
prebuilt pullable image instead. The legacy `OPENSHELL_DOCKER_SUPERVISOR_IMAGE`
and `OPENSHELL_SUPERVISOR_IMAGE` overrides remain supported and take precedence.
The pinned upstream checkout includes reference-client fixture drift that is
tracked in `modelcontextprotocol/conformance#345`. The wrapper patches the
+82
View File
@@ -9,6 +9,88 @@
# Keep an explicit override so telemetry-specific tests can opt back in.
export OPENSHELL_TELEMETRY_ENABLED="${OPENSHELL_TELEMETRY_ENABLED:-false}"
# Resolve a test image override. Repository-only values inherit the caller's
# tag, while tagged and digest-pinned references are already complete.
e2e_image_reference_is_complete() {
local image=$1
local last_component="${image##*/}"
[[ "${image}" == *@* || "${last_component}" == *:* ]]
}
e2e_image_reference_has_digest() {
[[ "$1" == *@* ]]
}
e2e_resolve_image_reference() {
local image=$1
local tag=$2
if e2e_image_reference_is_complete "${image}"; then
printf '%s\n' "${image}"
else
printf '%s:%s\n' "${image%/}" "${tag}"
fi
}
e2e_image_reference_repository() {
local image=$1
local repository="${image%%@*}"
local last_component="${repository##*/}"
if [[ "${last_component}" == *:* ]]; then
repository="${repository%:*}"
fi
printf '%s\n' "${repository}"
}
# Return the registry portion of an image repository. Docker treats the first
# path component as a registry when it contains a dot or colon, or is
# `localhost`; otherwise the image uses the configured/default registry.
e2e_image_reference_registry() {
local repository
local first_component
repository="$(e2e_image_reference_repository "$1")"
first_component="${repository%%/*}"
if [[ "${repository}" == */* ]] \
&& { [[ "${first_component}" == *.* ]] || [[ "${first_component}" == *:* ]] || [[ "${first_component}" == "localhost" ]]; }; then
printf '%s\n' "${first_component}"
fi
}
# Return the repository path without its registry, suitable for Helm's
# <component>.image.repository values.
e2e_image_reference_repository_path() {
local repository
local registry
repository="$(e2e_image_reference_repository "$1")"
registry="$(e2e_image_reference_registry "$1")"
if [ -n "${registry}" ]; then
printf '%s\n' "${repository#"${registry}"/}"
else
printf '%s\n' "${repository}"
fi
}
e2e_image_reference_tag() {
local image=$1
local repository="${image%%@*}"
local last_component="${repository##*/}"
if [[ "${image}" == *@* || "${last_component}" != *:* ]]; then
return 0
fi
printf '%s\n' "${last_component##*:}"
}
e2e_image_reference_digest() {
if [[ "$1" == *@* ]]; then
printf '%s\n' "${1#*@}"
fi
}
e2e_cargo_target_dir() {
local root=$1
shift
+20 -1
View File
@@ -17,6 +17,11 @@
# Sandbox image overrides:
# OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE=...
# OPENSHELL_E2E_DOCKER_SANDBOX_IMAGE_PULL_POLICY=always|if_not_present|never
# SANDBOX_IMAGE=... (trusted sandbox runtime override)
# Supervisor image overrides:
# SUPERVISOR_IMAGE=... (common test-wrapper override)
# OPENSHELL_SUPERVISOR_IMAGE=... (existing compatibility override)
# OPENSHELL_DOCKER_SUPERVISOR_IMAGE=... (Docker-specific override)
#
# The default sandbox image uses a mutable tag. This wrapper refreshes it
# before starting the gateway, while the Docker driver defaults to
@@ -323,6 +328,16 @@ resolve_docker_supervisor_image() {
return 0
fi
if [ -n "${SUPERVISOR_IMAGE:-}" ]; then
if [ -n "${CI:-}" ] && [ -z "${IMAGE_TAG:-}" ] \
&& ! e2e_image_reference_is_complete "${SUPERVISOR_IMAGE}"; then
echo "ERROR: IMAGE_TAG must be set in CI when SUPERVISOR_IMAGE is repository-only." >&2
exit 2
fi
printf '%s\n' "$(e2e_resolve_image_reference "${SUPERVISOR_IMAGE}" "${IMAGE_TAG:-dev}")"
return 0
fi
if [ -n "${CI:-}" ]; then
if [ -z "${IMAGE_TAG:-}" ]; then
echo "ERROR: IMAGE_TAG must be set in CI when no Docker supervisor image override is provided." >&2
@@ -347,6 +362,10 @@ resolve_docker_sandbox_runtime_image() {
printf '%s\n' "${OPENSHELL_SANDBOX_RUNTIME_IMAGE}"
return 0
fi
if [ -n "${SANDBOX_IMAGE:-}" ]; then
printf '%s\n' "$(e2e_resolve_image_reference "${SANDBOX_IMAGE}" "${IMAGE_TAG:-dev}")"
return 0
fi
if [ -n "${CI:-}" ]; then
if [ -z "${IMAGE_TAG:-}" ]; then
@@ -443,7 +462,7 @@ ensure_docker_supervisor_image() {
fi
echo "ERROR: supervisor image '${image}' is not available." >&2
echo " Build it, push it, or set OPENSHELL_SUPERVISOR_IMAGE to a pullable image." >&2
echo " Build it, push it, or set SUPERVISOR_IMAGE/OPENSHELL_SUPERVISOR_IMAGE to a pullable image." >&2
exit 2
}
+44 -24
View File
@@ -652,12 +652,9 @@ run_scenario() {
--namespace "${NAMESPACE}" --create-namespace \
"${helm_values_args[@]}" \
--set "fullnameOverride=openshell" \
--set "image.repository=${REGISTRY_VALUE}/gateway" \
--set "image.tag=${IMAGE_TAG_VALUE}" \
--set "sandboxRuntime.image.repository=${REGISTRY_VALUE}/sandbox" \
--set "sandboxRuntime.image.tag=${IMAGE_TAG_VALUE}" \
--set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \
--set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \
"${GATEWAY_HELM_IMAGE_ARGS[@]}" \
"${SUPERVISOR_HELM_IMAGE_ARGS[@]}" \
"${SANDBOX_RUNTIME_HELM_IMAGE_ARGS[@]}" \
"${helm_post_renderer_args[@]}" \
"$@" \
--wait --timeout 5m
@@ -940,6 +937,14 @@ else
IMAGE_TAG_VALUE="${IMAGE_TAG:-latest}"
fi
REGISTRY_VALUE="${REGISTRY_VALUE%/}"
GATEWAY_IMAGE="$(e2e_resolve_image_reference "${GATEWAY_IMAGE:-${REGISTRY_VALUE}/gateway}" "${IMAGE_TAG_VALUE}")"
SUPERVISOR_IMAGE="$(e2e_resolve_image_reference "${SUPERVISOR_IMAGE:-${REGISTRY_VALUE}/supervisor}" "${IMAGE_TAG_VALUE}")"
SANDBOX_RUNTIME_IMAGE="$(e2e_resolve_image_reference "${SANDBOX_IMAGE:-${REGISTRY_VALUE}/sandbox}" "${IMAGE_TAG_VALUE}")"
BUILD_GATEWAY_IMAGE="${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}"
BUILD_SUPERVISOR_IMAGE="${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}"
GATEWAY_HELM_IMAGE_ARGS=(--set-string "gateway.image.registry=$(e2e_image_reference_registry "${GATEWAY_IMAGE}")" --set-string "gateway.image.repository=$(e2e_image_reference_repository_path "${GATEWAY_IMAGE}")" --set-string "gateway.image.tag=$(e2e_image_reference_tag "${GATEWAY_IMAGE}")" --set-string "gateway.image.digest=$(e2e_image_reference_digest "${GATEWAY_IMAGE}")")
SUPERVISOR_HELM_IMAGE_ARGS=(--set-string "supervisor.image.registry=$(e2e_image_reference_registry "${SUPERVISOR_IMAGE}")" --set-string "supervisor.image.repository=$(e2e_image_reference_repository_path "${SUPERVISOR_IMAGE}")" --set-string "supervisor.image.tag=$(e2e_image_reference_tag "${SUPERVISOR_IMAGE}")" --set-string "supervisor.image.digest=$(e2e_image_reference_digest "${SUPERVISOR_IMAGE}")")
SANDBOX_RUNTIME_HELM_IMAGE_ARGS=(--set-string "sandboxRuntime.image.registry=$(e2e_image_reference_registry "${SANDBOX_RUNTIME_IMAGE}")" --set-string "sandboxRuntime.image.repository=$(e2e_image_reference_repository_path "${SANDBOX_RUNTIME_IMAGE}")" --set-string "sandboxRuntime.image.tag=$(e2e_image_reference_tag "${SANDBOX_RUNTIME_IMAGE}")" --set-string "sandboxRuntime.image.digest=$(e2e_image_reference_digest "${SANDBOX_RUNTIME_IMAGE}")")
# Resolve a host-gateway IP that sandbox pods can dial to reach test fixtures
# running on the developer/CI host (HTTP fixtures bound to 0.0.0.0 plus sibling
@@ -1032,7 +1037,7 @@ elif [[ "${KUBE_CONTEXT}" == k3d-* ]] && command -v k3d >/dev/null 2>&1; then
fi
if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then
require_cmd docker
echo "Building local Kubernetes e2e images (${REGISTRY_VALUE}/{gateway,sandbox,supervisor}:${IMAGE_TAG_VALUE})..."
echo "Building local Kubernetes e2e images (${BUILD_GATEWAY_IMAGE}, ${BUILD_SUPERVISOR_IMAGE})..."
if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then
if [ "$(uname -s)" != "Linux" ]; then
echo "ERROR: external Kubernetes driver image composition currently requires a Linux build host." >&2
@@ -1061,9 +1066,9 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then
cp "${external_driver}" "${external_stage}/openshell-driver-kubernetes"
docker build \
--build-arg "TARGETARCH=${external_arch}" \
--build-arg "SUPERVISOR_IMAGE=${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}" \
--build-arg "SUPERVISOR_IMAGE=${BUILD_SUPERVISOR_IMAGE}" \
--build-arg "SANDBOX_RUNTIME_IMAGE=${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}" \
--tag "${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \
--tag "${BUILD_GATEWAY_IMAGE}" \
--file "${ROOT}/e2e/docker/Dockerfile.external-kubernetes-gateway" \
"${ROOT}"
else
@@ -1071,7 +1076,11 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then
bash "${ROOT}/tasks/scripts/docker-build-image.sh" gateway
fi
sandbox_image="${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}"
supervisor_image="${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}"
if [ "${GATEWAY_IMAGE}" != "${BUILD_GATEWAY_IMAGE}" ]; then
if e2e_image_reference_has_digest "${GATEWAY_IMAGE}"; then echo "ERROR: digest-pinned GATEWAY_IMAGE requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=0" >&2; exit 2; fi
docker tag "${BUILD_GATEWAY_IMAGE}" "${GATEWAY_IMAGE}"
fi
supervisor_image="${BUILD_SUPERVISOR_IMAGE}"
if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" != "1" ] \
|| ! docker image inspect "${sandbox_image}" >/dev/null 2>&1; then
CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \
@@ -1080,6 +1089,13 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then
reuse_sandbox_image=1
echo "Reusing existing sandbox image ${sandbox_image}"
fi
if [ "${SANDBOX_RUNTIME_IMAGE}" != "${sandbox_image}" ]; then
if e2e_image_reference_has_digest "${SANDBOX_RUNTIME_IMAGE}"; then
echo "ERROR: digest-pinned SANDBOX_IMAGE requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=0" >&2
exit 2
fi
docker tag "${sandbox_image}" "${SANDBOX_RUNTIME_IMAGE}"
fi
if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" != "1" ] \
|| ! docker image inspect "${supervisor_image}" >/dev/null 2>&1; then
CONTAINER_ENGINE=docker IMAGE_REGISTRY="${REGISTRY_VALUE}" IMAGE_TAG="${IMAGE_TAG_VALUE}" \
@@ -1088,13 +1104,18 @@ if [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ]; then
reuse_supervisor_image=1
echo "Reusing existing supervisor image ${supervisor_image}"
fi
if [ "${SUPERVISOR_IMAGE}" != "${BUILD_SUPERVISOR_IMAGE}" ]; then
if e2e_image_reference_has_digest "${SUPERVISOR_IMAGE}"; then echo "ERROR: digest-pinned SUPERVISOR_IMAGE requires OPENSHELL_E2E_KUBE_BUILD_IMAGES=0" >&2; exit 2; fi
docker tag "${BUILD_SUPERVISOR_IMAGE}" "${SUPERVISOR_IMAGE}"
fi
fi
if [ -n "${import_cluster_name}" ]; then
for image in \
"${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}" \
"${GATEWAY_IMAGE}" \
"${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}" \
"${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}"; do
"${SUPERVISOR_IMAGE}" \
"${SANDBOX_RUNTIME_IMAGE}"; do
if docker image inspect "${image}" >/dev/null 2>&1; then
echo "Importing ${image} into k3d cluster ${import_cluster_name}..."
k3d image import "${image}" --cluster "${import_cluster_name}" \
@@ -1105,16 +1126,18 @@ elif [ "${OPENSHELL_E2E_KUBE_BUILD_IMAGES}" = "1" ] \
&& [[ "${KUBE_CONTEXT}" == kind-* ]] \
&& command -v kind >/dev/null 2>&1; then
kind_cluster_name="${KUBE_CONTEXT#kind-}"
kind_images=("${REGISTRY_VALUE}/gateway:${IMAGE_TAG_VALUE}")
kind_images=("${GATEWAY_IMAGE}")
# The CI workflow loads its published sandbox archive before invoking this
# wrapper. Only load a sandbox image here when this script rebuilt it.
if [ "${reuse_sandbox_image}" != "1" ]; then
kind_images+=("${REGISTRY_VALUE}/sandbox:${IMAGE_TAG_VALUE}")
# wrapper. Load a replacement only when this script rebuilt or retagged it.
if [ "${reuse_sandbox_image}" != "1" ] \
|| [ "${SANDBOX_RUNTIME_IMAGE}" != "${sandbox_image}" ]; then
kind_images+=("${SANDBOX_RUNTIME_IMAGE}")
fi
# The CI workflow loads its published supervisor archive before invoking this
# wrapper. Only load a supervisor image here when this script rebuilt it.
if [ "${reuse_supervisor_image}" != "1" ]; then
kind_images+=("${REGISTRY_VALUE}/supervisor:${IMAGE_TAG_VALUE}")
if [ "${reuse_supervisor_image}" != "1" ] \
|| [ "${SUPERVISOR_IMAGE}" != "${BUILD_SUPERVISOR_IMAGE}" ]; then
kind_images+=("${SUPERVISOR_IMAGE}")
fi
for image in "${kind_images[@]}"; do
echo "Loading ${image} into kind cluster ${kind_cluster_name}..."
@@ -1361,12 +1384,9 @@ else
--namespace "${NAMESPACE}" --create-namespace \
"${helm_values_args[@]}" \
--set "fullnameOverride=openshell" \
--set "image.repository=${REGISTRY_VALUE}/gateway" \
--set "image.tag=${IMAGE_TAG_VALUE}" \
--set "sandboxRuntime.image.repository=${REGISTRY_VALUE}/sandbox" \
--set "sandboxRuntime.image.tag=${IMAGE_TAG_VALUE}" \
--set "supervisor.image.repository=${REGISTRY_VALUE}/supervisor" \
--set "supervisor.image.tag=${IMAGE_TAG_VALUE}" \
"${GATEWAY_HELM_IMAGE_ARGS[@]}" \
"${SUPERVISOR_HELM_IMAGE_ARGS[@]}" \
"${SANDBOX_RUNTIME_HELM_IMAGE_ARGS[@]}" \
"${helm_extra_args[@]}" \
"${helm_post_renderer_args[@]}" \
--wait --timeout 5m
+30 -2
View File
@@ -14,6 +14,11 @@
# HTTPS endpoint-only mode is intentionally unsupported here. Use a named
# gateway config when mTLS materials are needed.
#
# Supervisor image overrides:
# SUPERVISOR_IMAGE=... (common test-wrapper override)
# OPENSHELL_SUPERVISOR_IMAGE=... (existing compatibility override)
# SANDBOX_IMAGE=... (trusted sandbox runtime override)
#
# Set OPENSHELL_E2E_PODMAN_STOP_TIMEOUT_SECS to override the managed gateway's
# Podman sandbox stop timeout. The harness default is intentionally shorter
# than the production driver default to keep CI teardown bounded.
@@ -346,6 +351,16 @@ resolve_podman_supervisor_image() {
return 0
fi
if [ -n "${SUPERVISOR_IMAGE:-}" ]; then
if [ -n "${CI:-}" ] && [ -z "${IMAGE_TAG:-}" ] \
&& ! e2e_image_reference_is_complete "${SUPERVISOR_IMAGE}"; then
echo "ERROR: IMAGE_TAG must be set in CI when SUPERVISOR_IMAGE is repository-only." >&2
exit 2
fi
printf '%s\n' "$(e2e_resolve_image_reference "${SUPERVISOR_IMAGE}" "${IMAGE_TAG:-dev}")"
return 0
fi
if [ -n "${CI:-}" ]; then
if [ -z "${IMAGE_TAG:-}" ]; then
echo "ERROR: IMAGE_TAG must be set in CI when no Podman supervisor image override is provided." >&2
@@ -365,6 +380,10 @@ resolve_podman_sandbox_runtime_image() {
printf '%s\n' "${OPENSHELL_SANDBOX_RUNTIME_IMAGE}"
return 0
fi
if [ -n "${SANDBOX_IMAGE:-}" ]; then
printf '%s\n' "$(e2e_resolve_image_reference "${SANDBOX_IMAGE}" "${IMAGE_TAG:-dev}")"
return 0
fi
if [ -n "${CI:-}" ]; then
if [ -z "${IMAGE_TAG:-}" ]; then
@@ -387,6 +406,11 @@ ensure_podman_supervisor_image() {
local dockerfile=${OPENSHELL_E2E_SUPERVISOR_DOCKERFILE:-${ROOT}/deploy/docker/Dockerfile.supervisor}
local context="${WORKDIR}/supervisor-image" arch
case "${image}" in
*@*)
echo "ERROR: supplied supervisor binaries cannot be built to a digest-pinned image reference: ${image}" >&2
echo " Use a tagged image reference when building from OPENSHELL_E2E_SUPERVISOR_BIN." >&2
exit 2
;;
*:dev|*:latest)
echo "ERROR: supplied supervisor binaries require a unique versioned image tag, not ${image}." >&2
exit 2
@@ -470,7 +494,7 @@ ensure_podman_supervisor_image() {
fi
echo "ERROR: supervisor image '${image}' is not available." >&2
echo " Build it, push it, or set OPENSHELL_SUPERVISOR_IMAGE to a pullable image." >&2
echo " Build it, push it, or set SUPERVISOR_IMAGE/OPENSHELL_SUPERVISOR_IMAGE to a pullable image." >&2
exit 2
}
@@ -577,7 +601,11 @@ fi
# isolated XDG store where this image was built. Address the local image by its
# immutable manifest digest so policy=missing cannot resolve a mutable tag or
# contact a registry for a different artifact.
SUPERVISOR_IMAGE_REPOSITORY="${SUPERVISOR_IMAGE%:*}"
SUPERVISOR_IMAGE_REPOSITORY="${SUPERVISOR_IMAGE%%@*}"
last_component="${SUPERVISOR_IMAGE_REPOSITORY##*/}"
if [[ "${last_component}" == *:* ]]; then
SUPERVISOR_IMAGE_REPOSITORY="${SUPERVISOR_IMAGE_REPOSITORY%:*}"
fi
SUPERVISOR_RUNTIME_IMAGE="${SUPERVISOR_IMAGE_REPOSITORY}@${SUPERVISOR_IMAGE_DIGEST}"
if ! [[ "${SUPERVISOR_RUNTIME_IMAGE}" =~ ^[^@]+@sha256:[0-9a-f]{64}$ ]]; then
echo "ERROR: supervisor runtime image is not digest-pinned: ${SUPERVISOR_RUNTIME_IMAGE}" >&2