mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-20 06:00:06 +08:00
krishicks
3dd5ce3314
The deterministic fixture still modeled Alpine after the supervisor image switched to Debian, causing the wrapper to exit before producing parity evidence. Update its image expectations and launch evidence to match the current harness contract. Force the intentional replacement of the read-only staged artifact so BSD mv does not prompt when the deterministic parity test runs from a terminal. Signed-off-by: Kris Hicks <khicks@nvidia.com>
584 lines
30 KiB
Bash
Executable File
584 lines
30 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Deterministic contract tests for e2e/parity/run.sh. No container runtime is
|
|
# invoked; the Podman wrapper and all three artifacts are tiny local fakes.
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
TEST_SUPERVISOR_BASE="$(awk '$1 == "FROM" { print $2; exit }' "${ROOT}/deploy/docker/Dockerfile.supervisor")"
|
|
TMP_ROOT="${TMPDIR:-/tmp}"
|
|
TMP_ROOT="${TMP_ROOT%/}"
|
|
WORKDIR="$(mktemp -d "${TMP_ROOT}/openshell-parity-test.XXXXXX")"
|
|
trap 'rm -rf "${WORKDIR}"' EXIT
|
|
|
|
fail() { echo "FAIL: $*" >&2; exit 1; }
|
|
assert_contains() { grep -F -- "$2" "$1" >/dev/null || fail "expected $1 to contain: $2"; }
|
|
assert_not_contains() { ! grep -F -- "$2" "$1" >/dev/null || fail "expected $1 not to contain: $2"; }
|
|
assert_status() { [ "$1" -eq "$2" ] || fail "expected status $2, got $1"; }
|
|
|
|
# Package provenance must work for both Debian status and distroless status.d.
|
|
uv run --no-project python - "${ROOT}" "${WORKDIR}" <<'PYTEST'
|
|
import runpy
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
manifest = runpy.run_path(str(Path(sys.argv[1]) / "e2e/support/debian-package-manifest.py"))["package_manifest"]
|
|
root = Path(sys.argv[2]) / "dpkg-fixture"
|
|
root.mkdir()
|
|
(root / "status").write_text("Package: removed\nStatus: deinstall ok config-files\n\n")
|
|
(root / "status.d").mkdir()
|
|
(root / "status.d/libc6").write_text("Package: libc6\nVersion: 2.41\nArchitecture: arm64\nMulti-Arch: same\n")
|
|
(root / "status.d/libc6.md5sums").write_text("ignored checksum file")
|
|
assert manifest(root) == ["libc6:arm64=2.41"]
|
|
(root / "status").write_text("Package: ca-certificates\nStatus: install ok installed\nVersion: 20250419\nArchitecture: all\n")
|
|
assert manifest(root) == ["ca-certificates=20250419", "libc6:arm64=2.41"]
|
|
(root / "status.d/libc6").write_text("Package: broken\n")
|
|
try:
|
|
manifest(root)
|
|
except ValueError:
|
|
pass
|
|
else:
|
|
raise AssertionError("incomplete package metadata accepted")
|
|
try:
|
|
manifest(root / "missing")
|
|
except ValueError:
|
|
pass
|
|
else:
|
|
raise AssertionError("missing package metadata accepted")
|
|
PYTEST
|
|
|
|
# Schema generator behavior is separately deterministic and does not need a
|
|
# gateway, certificates, or Podman.
|
|
# shellcheck source=e2e/support/gateway-common.sh
|
|
source "${ROOT}/e2e/support/gateway-common.sh"
|
|
# shellcheck source=e2e/support/podman-gateway-config.sh
|
|
source "${ROOT}/e2e/support/podman-gateway-config.sh"
|
|
mkdir -p "${WORKDIR}/pki/client" "${WORKDIR}/jwt"
|
|
e2e_write_podman_gateway_config "${WORKDIR}/v1.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 ''
|
|
e2e_write_podman_gateway_config "${WORKDIR}/v2.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test '' '' 0 ''
|
|
assert_contains "${WORKDIR}/v1.toml" 'version = 1'
|
|
assert_contains "${WORKDIR}/v1.toml" 'compute_drivers = ["podman"]'
|
|
assert_contains "${WORKDIR}/v1.toml" 'image_pull_policy = "missing"'
|
|
assert_contains "${WORKDIR}/v1.toml" 'health_check_interval_secs = 0'
|
|
assert_contains "${WORKDIR}/v1.toml" 'guest_tls_ca = '
|
|
assert_contains "${WORKDIR}/v2.toml" 'version = 2'
|
|
assert_contains "${WORKDIR}/v2.toml" 'compute_driver = "podman"'
|
|
assert_contains "${WORKDIR}/v2.toml" 'image_pull_policy = "if_not_present"'
|
|
assert_not_contains "${WORKDIR}/v2.toml" 'health_check_interval_secs = 0'
|
|
# V2 guest TLS is emitted before its driver table; V1 is driver-local.
|
|
OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v1-options.toml" 1 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 ""
|
|
OPENSHELL_E2E_PODMAN_OPTION_PROFILE=podman-options e2e_write_podman_gateway_config "${WORKDIR}/v2-options.toml" 2 "${ROOT}" "${WORKDIR}/pki" "${WORKDIR}/jwt" test-gateway 0 socket network 18181 image:test 15 supervisor:test "" "" 0 ""
|
|
for config in "${WORKDIR}/v1-options.toml" "${WORKDIR}/v2-options.toml"; do
|
|
assert_contains "${config}" 'sandbox_pids_limit = 31'
|
|
assert_contains "${config}" 'health_check_interval_secs = 7'
|
|
assert_not_contains "${config}" 'app_armor_profile = '
|
|
done
|
|
assert_contains "${WORKDIR}/v1-options.toml" 'sandbox_ssh_socket_path = "/run/openshell/parity-ssh.sock"'
|
|
assert_contains "${WORKDIR}/v2-options.toml" 'ssh_socket_path = "/run/openshell/parity-ssh.sock"'
|
|
if OPENSHELL_E2E_PODMAN_OPTION_PROFILE=unknown e2e_podman_option_profile >/dev/null 2>&1; then fail 'unknown option profile unexpectedly accepted'; fi
|
|
|
|
v1_driver_line="$(grep -n '^\[openshell.drivers.podman\]' "${WORKDIR}/v1.toml" | cut -d: -f1)"
|
|
v1_tls_line="$(grep -n '^guest_tls_ca' "${WORKDIR}/v1.toml" | cut -d: -f1)"
|
|
v2_driver_line="$(grep -n '^\[openshell.drivers.podman\]' "${WORKDIR}/v2.toml" | cut -d: -f1)"
|
|
v2_tls_line="$(grep -n '^guest_tls_ca' "${WORKDIR}/v2.toml" | cut -d: -f1)"
|
|
[ "${v1_tls_line}" -gt "${v1_driver_line}" ] || fail 'v1 TLS must be driver-local'
|
|
[ "${v2_tls_line}" -lt "${v2_driver_line}" ] || fail 'v2 TLS must be gateway-owned'
|
|
if OPENSHELL_E2E_CONFIG_SCHEMA_VERSION=3 e2e_podman_config_schema_version >/dev/null 2>&1; then
|
|
fail 'invalid schema version unexpectedly accepted'
|
|
fi
|
|
set +e
|
|
env -u OPENSHELL_GATEWAY_ENDPOINT \
|
|
OPENSHELL_E2E_CONFIG_SCHEMA_VERSION=3 \
|
|
bash "${ROOT}/e2e/with-podman-gateway.sh" true >"${WORKDIR}/wrapper-schema.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 2
|
|
assert_contains "${WORKDIR}/wrapper-schema.out" 'must be 1 or 2'
|
|
|
|
cat >"${WORKDIR}/trace-cli-fixture" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
printf 'openshell-conformance-tracefixture\n'
|
|
printf 'trace fixture stderr\n' >&2
|
|
EOF
|
|
chmod +x "${WORKDIR}/trace-cli-fixture"
|
|
OPENSHELL_PARITY_REAL_CLI="${WORKDIR}/trace-cli-fixture" \
|
|
OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE="${WORKDIR}/trace-cli.stdout" \
|
|
bash "${ROOT}/e2e/parity/trace-cli.sh" sandbox exec -- echo marker \
|
|
>"${WORKDIR}/trace-cli.forwarded.stdout" \
|
|
2>"${WORKDIR}/trace-cli.forwarded.stderr"
|
|
cmp -s "${WORKDIR}/trace-cli.stdout" "${WORKDIR}/trace-cli.forwarded.stdout" \
|
|
|| fail 'CLI trace wrapper did not preserve exact exec stdout'
|
|
assert_contains "${WORKDIR}/trace-cli.forwarded.stderr" 'trace fixture stderr'
|
|
|
|
HEAD_SHA="$(git -C "${ROOT}" rev-parse HEAD)"
|
|
cat >"${WORKDIR}/manifest.toml" <<EOF
|
|
manifest_version = 1
|
|
baseline_ref = "origin/main"
|
|
baseline_commit = "${HEAD_SHA}"
|
|
EOF
|
|
mkdir -p "${WORKDIR}/bin"
|
|
cat >"${WORKDIR}/bin/fake-wrapper" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
for variable in \
|
|
OPENSHELL_GATEWAY_ENDPOINT OPENSHELL_GATEWAY_CONFIG OPENSHELL_COMPUTE_DRIVER \
|
|
OPENSHELL_COMPUTE_DRIVER_SOCKET OPENSHELL_DRIVERS OPENSHELL_PODMAN_SOCKET \
|
|
CONTAINER_HOST CONTAINER_CONNECTION CONTAINERS_STORAGE_CONF CONTAINERS_CONF \
|
|
CONTAINERS_REGISTRIES_CONF CONTAINERS_REGISTRIES_CONF_DIR CONTAINERS_POLICY \
|
|
PODMAN_CONNECTIONS_CONF DOCKER_HOST OPENSHELL_SANDBOX_IMAGE \
|
|
OPENSHELL_SANDBOX_RUNTIME_IMAGE \
|
|
OPENSHELL_GRPC_ENDPOINT OPENSHELL_PODMAN_HOST_GATEWAY_IP OPENSHELL_PODMAN_USERNS \
|
|
OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET \
|
|
OPENSHELL_APP_ARMOR_PROFILE OPENSHELL_SANDBOX_HTTPS_PROXY OPENSHELL_SANDBOX_NO_PROXY \
|
|
OPENSHELL_SANDBOX_PROXY_AUTH_FILE OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE \
|
|
OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME OPENSHELL_SANDBOX_PROXY_CA_BUNDLE \
|
|
OPENSHELL_OTLP_ENDPOINT OPENSHELL_GATEWAY_NAME OPENSHELL_COMPUTE_DRIVER_BIND; do
|
|
[ -z "${!variable:-}" ] || exit 23
|
|
done
|
|
expected_sandbox="ghcr.io/nvidia/openshell-community/sandboxes/base@sha256:$(printf '%064d' 0)"
|
|
[ "${OPENSHELL_E2E_REQUIRE_DIGEST_PINNED_SANDBOX_IMAGE:-0}" = 1 ] || exit 24
|
|
[ "${OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE:-}" = "${expected_sandbox}" ] || exit 25
|
|
[ "${OPENSHELL_COMMUNITY_REGISTRY:-}" = "ghcr.io/nvidia/openshell-community/sandboxes" ] || exit 28
|
|
expected_base="docker.io/library/debian@sha256:$(printf '%064d' 0)"
|
|
[ "${OPENSHELL_E2E_SUPERVISOR_BASE_IMAGE:-}" = "${OPENSHELL_PARITY_TEST_SUPERVISOR_BASE}" ] || exit 26
|
|
[ "${OPENSHELL_E2E_SUPERVISOR_BASE_RUNTIME_IMAGE:-}" = "${expected_base}" ] || exit 27
|
|
printf '%s|%s|%s|%s|%s|%s|%s|%s|%s|%s|%s\n' "$OPENSHELL_PARITY_VARIANT" "$OPENSHELL_E2E_CONFIG_SCHEMA_VERSION" "$OPENSHELL_GATEWAY_BIN" "$OPENSHELL_BIN" "$OPENSHELL_CONFORMANCE_BIN" "$MISE_TRUSTED_CONFIG_PATHS" "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" "${OPENSHELL_PARITY_ORACLE_RESULT:-}" "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-}" "${OPENSHELL_EXTERNAL_DRIVER_BIN:-}" "${OPENSHELL_E2E_SUPERVISOR_BIN:-}" >>"$OPENSHELL_PARITY_TEST_CALLS"
|
|
mkdir -p "$XDG_DATA_HOME/containers/storage"
|
|
printf 'fixture-package-1.0-r0\n' >"${OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE}"
|
|
package_hash="$(sha256sum "${OPENSHELL_PARITY_SUPERVISOR_PACKAGE_CAPTURE}" | cut -d' ' -f1)"
|
|
OPENSHELL_PARITY_FIXTURE_PACKAGE_HASH="${package_hash}" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
variant = os.environ["OPENSHELL_PARITY_VARIANT"]
|
|
schema = int(os.environ["OPENSHELL_E2E_CONFIG_SCHEMA_VERSION"])
|
|
external = os.environ.get("OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER") == "1"
|
|
zero = "0" * 64
|
|
image_digest = f"sha256:{zero}"
|
|
sandbox_runtime = f"ghcr.io/nvidia/openshell-community/sandboxes/base@{image_digest}"
|
|
sandbox_boundary = "localhost/openshell/sandbox:dev"
|
|
supervisor_runtime = f"localhost/openshell/supervisor@{image_digest}"
|
|
base_runtime = f"docker.io/library/debian@{image_digest}"
|
|
pull_policy = "missing" if schema == 1 else "if_not_present"
|
|
gateway_port = 18181
|
|
callback = f"https://host.containers.internal:{gateway_port}"
|
|
driver_socket = f"/tmp/{variant}-driver.sock"
|
|
podman_socket = f"/tmp/{variant}-podman.sock"
|
|
network = f"{variant}-network"
|
|
|
|
selector = (
|
|
'compute_drivers = ["podman"]'
|
|
if schema == 1
|
|
else 'compute_driver = "podman"'
|
|
)
|
|
config_lines = [
|
|
"[openshell]",
|
|
f"version = {schema}",
|
|
"[openshell.gateway]",
|
|
selector,
|
|
"[openshell.drivers.podman]",
|
|
f'socket_path = "{driver_socket}"',
|
|
]
|
|
if not external:
|
|
config_lines.extend(
|
|
[
|
|
f'network_name = "{network}"',
|
|
f'default_image = "{sandbox_runtime}"',
|
|
f'image_pull_policy = "{pull_policy}"',
|
|
f'supervisor_image = "{supervisor_runtime}"',
|
|
]
|
|
)
|
|
Path(os.environ["OPENSHELL_PARITY_GATEWAY_CONFIG_CAPTURE"]).write_text(
|
|
"\n".join(config_lines) + "\n", encoding="utf-8"
|
|
)
|
|
|
|
launch = {
|
|
"schema_version": schema,
|
|
"gateway_port": gateway_port,
|
|
"external_compute_driver": external,
|
|
"compute_driver_transport": "remote_uds" if external else "in_tree",
|
|
"external_driver_pull_policy": pull_policy,
|
|
"supervisor_image": os.environ["OPENSHELL_SUPERVISOR_IMAGE"],
|
|
"supervisor_image_id": zero,
|
|
"supervisor_image_digest": image_digest,
|
|
"supervisor_runtime_image": supervisor_runtime,
|
|
"supervisor_base_image": os.environ["OPENSHELL_PARITY_TEST_SUPERVISOR_BASE"],
|
|
"supervisor_base_image_id": zero,
|
|
"supervisor_base_image_digest": image_digest,
|
|
"supervisor_base_runtime_image": base_runtime,
|
|
"supervisor_package_manifest_sha256": os.environ[
|
|
"OPENSHELL_PARITY_FIXTURE_PACKAGE_HASH"
|
|
],
|
|
"sandbox_image_request": sandbox_runtime,
|
|
"sandbox_image_id": zero,
|
|
"sandbox_image_digest": image_digest,
|
|
"sandbox_runtime_image": sandbox_runtime,
|
|
"sandbox_boundary_image": sandbox_boundary,
|
|
"sandbox_client_image_alias": "ghcr.io/nvidia/openshell-community/sandboxes/base:latest",
|
|
"sandbox_client_image_alias_id": zero,
|
|
"gateway_sha256_before_execution": os.environ[
|
|
"OPENSHELL_E2E_EXPECTED_GATEWAY_SHA256"
|
|
],
|
|
"cli_sha256_before_execution": os.environ["OPENSHELL_E2E_EXPECTED_CLI_SHA256"],
|
|
"conformance_sha256_before_execution": os.environ[
|
|
"OPENSHELL_E2E_EXPECTED_CONFORMANCE_SHA256"
|
|
],
|
|
"external_driver_sha256_before_execution": os.environ.get(
|
|
"OPENSHELL_E2E_EXPECTED_EXTERNAL_DRIVER_SHA256", ""
|
|
),
|
|
"supervisor_sha256_before_execution": os.environ[
|
|
"OPENSHELL_E2E_EXPECTED_SUPERVISOR_SHA256"
|
|
],
|
|
"supervisor_dockerfile_sha256_before_execution": os.environ[
|
|
"OPENSHELL_E2E_EXPECTED_SUPERVISOR_DOCKERFILE_SHA256"
|
|
],
|
|
"cli_trace_wrapper_sha256_before_execution": os.environ[
|
|
"OPENSHELL_E2E_EXPECTED_CLI_TRACE_WRAPPER_SHA256"
|
|
],
|
|
}
|
|
if external:
|
|
launch.update(
|
|
{
|
|
"external_driver_grpc_endpoint": callback,
|
|
"external_driver_host_gateway_ip": "host-gateway",
|
|
"external_driver_userns": None,
|
|
"external_driver_spiffe": False,
|
|
"external_driver_proxy": False,
|
|
"external_driver_app_armor": False,
|
|
"external_driver_environment": {
|
|
"XDG_DATA_HOME": f"/tmp/{variant}-driver-data",
|
|
"OPENSHELL_COMPUTE_DRIVER_SOCKET": driver_socket,
|
|
"OPENSHELL_PODMAN_SOCKET": podman_socket,
|
|
"OPENSHELL_SANDBOX_IMAGE": sandbox_runtime,
|
|
"OPENSHELL_SANDBOX_IMAGE_PULL_POLICY": pull_policy,
|
|
"OPENSHELL_SANDBOX_RUNTIME_IMAGE": sandbox_boundary,
|
|
"OPENSHELL_HEALTH_CHECK_INTERVAL_SECS": 10,
|
|
"OPENSHELL_GRPC_ENDPOINT": callback,
|
|
"OPENSHELL_GATEWAY_PORT": gateway_port,
|
|
"OPENSHELL_NETWORK_NAME": network,
|
|
"OPENSHELL_STOP_TIMEOUT": 15,
|
|
"OPENSHELL_SUPERVISOR_IMAGE": supervisor_runtime,
|
|
"OPENSHELL_PODMAN_TLS_CA": {
|
|
"path": f"/tmp/{variant}-pki/ca.crt",
|
|
"sha256": "8" * 64,
|
|
},
|
|
"OPENSHELL_PODMAN_TLS_CERT": {
|
|
"path": f"/tmp/{variant}-pki/tls.crt",
|
|
"sha256": "9" * 64,
|
|
},
|
|
"OPENSHELL_PODMAN_TLS_KEY": {
|
|
"path": f"/tmp/{variant}-pki/tls.key",
|
|
"sha256": "a" * 64,
|
|
},
|
|
"OPENSHELL_ENABLE_BIND_MOUNTS": True,
|
|
},
|
|
}
|
|
)
|
|
Path(os.environ["OPENSHELL_PARITY_LAUNCH_MANIFEST_CAPTURE"]).write_text(
|
|
json.dumps(launch, separators=(",", ":")) + "\n", encoding="utf-8"
|
|
)
|
|
PY
|
|
run_id="fixture${OPENSHELL_PARITY_VARIANT}"
|
|
printf 'openshell-conformance-%s\n' "${run_id}" >"${OPENSHELL_PARITY_EXEC_STDOUT_CAPTURE}"
|
|
printf 'CLI conformance run ID: %s\n' "${run_id}" >&2
|
|
printf 'gateway preflight connected: gateway=fixture, authentication=authenticated\n' >&2
|
|
for marker in status create get-ready list-visible/0 exec delete list-empty/query/0; do
|
|
printf '[run %s][smoke/%s] completed in 1ms: exit 0\n' "${run_id}" "${marker}" >&2
|
|
done
|
|
printf '%s %064d sha256:%064d %s %s %s %s\n' \
|
|
"${expected_sandbox}" 0 0 "${expected_base}" \
|
|
"localhost/openshell/supervisor@sha256:$(printf '%064d' 0)" \
|
|
"ghcr.io/nvidia/openshell-community/sandboxes/base:latest" \
|
|
"${package_hash}" >&2
|
|
if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = 1 ]; then
|
|
printf 'fixture external driver log\n' >"${OPENSHELL_PARITY_EXTERNAL_DRIVER_LOG_CAPTURE}"
|
|
fi
|
|
if [ "${OPENSHELL_PARITY_TEST_MUTATE_ARTIFACT:-}" = "${OPENSHELL_PARITY_VARIANT}" ]; then
|
|
replacement="${OPENSHELL_GATEWAY_BIN}.replacement"
|
|
printf '#!/usr/bin/env bash\nexit 0\n# mutated\n' >"${replacement}"
|
|
chmod 0555 "${replacement}"
|
|
mv -f "${replacement}" "${OPENSHELL_GATEWAY_BIN}"
|
|
fi
|
|
if [ "${OPENSHELL_E2E_PODMAN_OPTION_PROFILE:-}" = podman-options ]; then
|
|
case "${OPENSHELL_PARITY_VARIANT}" in baseline) pids=2048 ;; candidate) pids=31 ;; esac
|
|
stable=true
|
|
if [ "${OPENSHELL_PARITY_TEST_SEMANTIC_DRIFT:-0}" = 1 ] && [ "${OPENSHELL_PARITY_VARIANT}" = candidate ]; then stable=false; fi
|
|
if [ "${OPENSHELL_PARITY_TEST_SKIP_RESULT:-}" != "${OPENSHELL_PARITY_VARIANT}" ]; then
|
|
printf '%s\n' "{\"scenario\":\"podman-options\",\"stable\":${stable},\"pids_limit\":${pids}}" > "${OPENSHELL_PARITY_ORACLE_RESULT}"
|
|
fi
|
|
fi
|
|
exec "$@"
|
|
EOF
|
|
cat >"${WORKDIR}/bin/fake-podman" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
printf '%s\n' "$*" >>"$OPENSHELL_PARITY_TEST_PODMAN_CALLS"
|
|
case "$1" in
|
|
pull) exit 0 ;;
|
|
image)
|
|
[ "$2" = inspect ] || exit 19
|
|
case "$4" in
|
|
'{{.Id}}') printf 'sha256:%064d\n' 0 ;;
|
|
'{{.Digest}}') printf 'sha256:%064d\n' 0 ;;
|
|
'{{index .RepoDigests 0}}') printf 'docker.io/library/debian@sha256:%064d\n' 0 ;;
|
|
*) exit 19 ;;
|
|
esac
|
|
;;
|
|
unshare)
|
|
shift
|
|
exec "$@"
|
|
;;
|
|
*) exit 19 ;;
|
|
esac
|
|
EOF
|
|
cat >"${WORKDIR}/bin/fake-conformance" <<'EOF'
|
|
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
if [ "${OPENSHELL_PARITY_FAIL_VARIANT:-}" = "${OPENSHELL_PARITY_VARIANT:-}" ] || [ "${OPENSHELL_PARITY_FAIL_VARIANT:-}" = both ]; then
|
|
exit 17
|
|
fi
|
|
if [ "${OPENSHELL_PARITY_TEST_INVALID_REPORT:-0}" = 1 ]; then
|
|
# Prove the verifier parses retained stdout instead of accepting a success
|
|
# substring injected into unrelated raw stderr.
|
|
printf '%s\n' '"passed": true' >&2
|
|
printf '%s\n' '{"scenarios":[{"name":"smoke","passed":false,"diagnostic":"fixture failure"}],"passed":false}'
|
|
else
|
|
printf '%s\n' '{'
|
|
printf '%s\n' ' "scenarios": [{"name":"smoke","passed":true,"diagnostic":null}],'
|
|
printf '%s\n' ' "passed": true'
|
|
printf '%s\n' '}'
|
|
fi
|
|
EOF
|
|
for artifact in baseline-gateway baseline-cli candidate-gateway candidate-cli baseline-driver candidate-driver baseline-supervisor candidate-supervisor; do
|
|
printf '#!/usr/bin/env bash\n# %s\nexit 0\n' "${artifact}" >"${WORKDIR}/bin/${artifact}"
|
|
done
|
|
chmod +x "${WORKDIR}/bin/"*
|
|
|
|
# shellcheck source=e2e/support/podman-gateway-config.sh
|
|
source "${ROOT}/e2e/support/podman-gateway-config.sh"
|
|
[ "$(e2e_podman_external_driver_pull_policy 1)" = missing ] || fail 'schema v1 external pull policy mismatch'
|
|
[ "$(e2e_podman_external_driver_pull_policy 2)" = if_not_present ] || fail 'schema v2 external pull policy mismatch'
|
|
|
|
run_harness() {
|
|
if [ "${OPENSHELL_PARITY_TEST_KEEP_RESULTS_DIR:-0}" != 1 ]; then
|
|
rm -rf -- "${WORKDIR}/results"
|
|
fi
|
|
OPENSHELL_PARITY_TEST_SUPERVISOR_BASE="${TEST_SUPERVISOR_BASE}" \
|
|
OPENSHELL_PARITY_CAPABILITY_MANIFEST="${WORKDIR}/manifest.toml" \
|
|
OPENSHELL_PARITY_BASELINE_WORKTREE="${ROOT}" \
|
|
OPENSHELL_PARITY_PODMAN_WRAPPER="${WORKDIR}/bin/fake-wrapper" \
|
|
OPENSHELL_PARITY_PODMAN_BIN="${WORKDIR}/bin/fake-podman" \
|
|
OPENSHELL_PARITY_PODMAN_OPTIONS_ORACLE="${WORKDIR}/bin/fake-conformance" \
|
|
OPENSHELL_PARITY_BASELINE_GATEWAY_BIN="${WORKDIR}/bin/baseline-gateway" \
|
|
OPENSHELL_PARITY_BASELINE_CLI_BIN="${WORKDIR}/bin/baseline-cli" \
|
|
OPENSHELL_PARITY_BASELINE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \
|
|
OPENSHELL_PARITY_CANDIDATE_GATEWAY_BIN="${WORKDIR}/bin/candidate-gateway" \
|
|
OPENSHELL_PARITY_CANDIDATE_CLI_BIN="${WORKDIR}/bin/candidate-cli" \
|
|
OPENSHELL_PARITY_CANDIDATE_CONFORMANCE_BIN="${WORKDIR}/bin/fake-conformance" \
|
|
OPENSHELL_PARITY_BASELINE_EXTERNAL_DRIVER_BIN="${WORKDIR}/bin/baseline-driver" \
|
|
OPENSHELL_PARITY_CANDIDATE_EXTERNAL_DRIVER_BIN="${OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE:-${WORKDIR}/bin/candidate-driver}" \
|
|
OPENSHELL_PARITY_BASELINE_SUPERVISOR_BIN="${WORKDIR}/bin/baseline-supervisor" \
|
|
OPENSHELL_PARITY_CANDIDATE_SUPERVISOR_BIN="${WORKDIR}/bin/candidate-supervisor" \
|
|
OPENSHELL_PARITY_RESULTS_DIR="${WORKDIR}/results" \
|
|
OPENSHELL_PARITY_TEST_CALLS="${WORKDIR}/calls" \
|
|
OPENSHELL_PARITY_TEST_PODMAN_CALLS="${WORKDIR}/podman-calls" \
|
|
MISE_TRUSTED_CONFIG_PATHS= \
|
|
bash "${ROOT}/e2e/parity/run.sh" --driver podman "$@"
|
|
}
|
|
|
|
OPENSHELL_GATEWAY_ENDPOINT=http://127.0.0.1:9 \
|
|
OPENSHELL_GATEWAY_CONFIG=/tmp/untrusted.toml \
|
|
OPENSHELL_COMPUTE_DRIVER=wrong \
|
|
OPENSHELL_COMPUTE_DRIVER_SOCKET=/tmp/untrusted.sock \
|
|
OPENSHELL_DRIVERS=wrong \
|
|
OPENSHELL_PODMAN_SOCKET=/tmp/untrusted-podman.sock \
|
|
CONTAINER_HOST=tcp://untrusted.invalid:9999 \
|
|
CONTAINER_CONNECTION=untrusted \
|
|
CONTAINERS_STORAGE_CONF=/tmp/untrusted-storage.conf \
|
|
CONTAINERS_CONF=/tmp/untrusted-containers.conf \
|
|
CONTAINERS_REGISTRIES_CONF=/tmp/untrusted-registries.conf \
|
|
CONTAINERS_REGISTRIES_CONF_DIR=/tmp/untrusted-registries.d \
|
|
CONTAINERS_POLICY=/tmp/untrusted-policy.json \
|
|
PODMAN_CONNECTIONS_CONF=/tmp/untrusted-connections.json \
|
|
DOCKER_HOST=tcp://untrusted.invalid:2375 \
|
|
OPENSHELL_SANDBOX_IMAGE=untrusted.invalid/sandbox:latest \
|
|
OPENSHELL_SANDBOX_RUNTIME_IMAGE=untrusted.invalid/runtime:latest \
|
|
OPENSHELL_GRPC_ENDPOINT=http://untrusted.invalid:1 \
|
|
OPENSHELL_PODMAN_HOST_GATEWAY_IP=192.0.2.1 \
|
|
OPENSHELL_PODMAN_USERNS=keep-id \
|
|
OPENSHELL_PROVIDER_SPIFFE_WORKLOAD_API_SOCKET=/tmp/untrusted-spiffe.sock \
|
|
OPENSHELL_E2E_PROVIDER_SPIFFE_SOCKET=/tmp/untrusted-e2e-spiffe.sock \
|
|
OPENSHELL_APP_ARMOR_PROFILE=Unconfined \
|
|
OPENSHELL_SANDBOX_HTTPS_PROXY=http://untrusted.invalid:8080 \
|
|
OPENSHELL_SANDBOX_NO_PROXY=untrusted.invalid \
|
|
OPENSHELL_SANDBOX_PROXY_AUTH_FILE=/tmp/untrusted-proxy-auth \
|
|
OPENSHELL_SANDBOX_PROXY_AUTH_ALLOW_INSECURE=true \
|
|
OPENSHELL_SANDBOX_PROXY_CONNECT_BY_HOSTNAME=true \
|
|
OPENSHELL_SANDBOX_PROXY_CA_BUNDLE=/tmp/untrusted-proxy-ca \
|
|
OPENSHELL_OTLP_ENDPOINT=http://untrusted.invalid:4317 \
|
|
OPENSHELL_GATEWAY_NAME=untrusted \
|
|
OPENSHELL_COMPUTE_DRIVER_BIND=192.0.2.2:50061 \
|
|
OPENSHELL_COMMUNITY_REGISTRY=untrusted.invalid/community \
|
|
run_harness
|
|
assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance"
|
|
assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance"
|
|
assert_contains "${WORKDIR}/calls" "|${ROOT}"
|
|
[ "$(sed -n '1s/|.*//p' "${WORKDIR}/calls")" = baseline ] || fail 'baseline was not invoked first'
|
|
[ "$(sed -n '2s/|.*//p' "${WORKDIR}/calls")" = candidate ] || fail 'candidate was not invoked second'
|
|
assert_contains "${WORKDIR}/results/baseline.json" "\"source_sha\":\"${HEAD_SHA}\""
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"schema_version":1'
|
|
assert_contains "${WORKDIR}/results/candidate.json" '"schema_version":2'
|
|
assert_contains "${WORKDIR}/results/candidate.json" "\"source_sha\":\"${HEAD_SHA}\""
|
|
assert_contains "${WORKDIR}/results/candidate.json" '"success":true'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"parity":true'
|
|
assert_contains "${WORKDIR}/results/semantic-verification.json" '"accepted": true'
|
|
assert_not_contains "${WORKDIR}/results/baseline.json" '"scenarios"'
|
|
assert_contains "${WORKDIR}/results/baseline.log" '"scenarios"'
|
|
assert_contains "${WORKDIR}/results/baseline.conformance.json" '"passed":true'
|
|
assert_contains "${WORKDIR}/podman-calls" 'pull ghcr.io/nvidia/openshell-community/sandboxes/base:latest'
|
|
assert_contains "${WORKDIR}/podman-calls" "pull ${TEST_SUPERVISOR_BASE}"
|
|
assert_contains "${WORKDIR}/podman-calls" 'unshare rm -rf -- '
|
|
assert_contains "${WORKDIR}/podman-calls" 'openshell-parity-run.'
|
|
|
|
set +e
|
|
OPENSHELL_PARITY_TEST_INVALID_REPORT=1 run_harness >"${WORKDIR}/invalid-report.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 1
|
|
assert_contains "${WORKDIR}/invalid-report.out" 'conformance report did not pass'
|
|
assert_contains "${WORKDIR}/invalid-report.out" 'semantic verification failed for smoke'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"accepted":false'
|
|
|
|
set +e
|
|
OPENSHELL_E2E_PODMAN_SANDBOX_IMAGE=untrusted.invalid/sandbox:latest \
|
|
run_harness >"${WORKDIR}/mutable-sandbox.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 2
|
|
assert_contains "${WORKDIR}/mutable-sandbox.out" 'must be digest-pinned for parity runs'
|
|
|
|
run_harness --scenario external-driver
|
|
assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/results/artifacts/baseline/external-driver|${WORKDIR}/results/artifacts/baseline/supervisor"
|
|
assert_contains "${WORKDIR}/calls" "|1|${WORKDIR}/results/artifacts/candidate/external-driver|${WORKDIR}/results/artifacts/candidate/supervisor"
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"scenario":"external-driver"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"command_class":"external_driver_conformance_smoke"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"gateway_profile":"driver-free"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"gateway_cargo_features":"--no-default-features --features telemetry"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"gateway_origin":"supplied_override"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_origin":"supplied_override"'
|
|
assert_contains "${WORKDIR}/results/baseline.launch.json" '"compute_driver_transport":"remote_uds"'
|
|
assert_contains "${WORKDIR}/results/baseline.launch.json" '"external_driver_pull_policy":"missing"'
|
|
assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_image_digest":"sha256:'
|
|
assert_contains "${WORKDIR}/results/baseline.launch.json" '"supervisor_runtime_image":"localhost/openshell/supervisor@sha256:'
|
|
assert_contains "${WORKDIR}/results/candidate.launch.json" '"external_driver_pull_policy":"if_not_present"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"gateway_sha256"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"cli_sha256"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"conformance_sha256"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"supervisor_origin":"supplied_override"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"supervisor_sha256"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"supervisor_dockerfile_sha256"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"external_driver_sha256"'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"classification":"pass"'
|
|
assert_contains "${WORKDIR}/results/semantic-verification.json" '"scenario": "external-driver"'
|
|
|
|
set +e
|
|
OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE="${WORKDIR}/bin/baseline-driver" \
|
|
run_harness --scenario external-driver >"${WORKDIR}/same-driver.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 2
|
|
assert_contains "${WORKDIR}/same-driver.out" 'requires distinct baseline and candidate driver artifacts'
|
|
|
|
cp "${WORKDIR}/bin/baseline-driver" "${WORKDIR}/bin/same-content-driver"
|
|
set +e
|
|
OPENSHELL_PARITY_TEST_CANDIDATE_DRIVER_OVERRIDE="${WORKDIR}/bin/same-content-driver" \
|
|
run_harness --scenario external-driver >"${WORKDIR}/same-driver-content.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 2
|
|
assert_contains "${WORKDIR}/same-driver-content.out" 'requires different baseline and candidate driver content'
|
|
|
|
run_harness --scenario podman-options
|
|
assert_contains "${WORKDIR}/calls" "baseline|1|${WORKDIR}/results/artifacts/baseline/gateway|${WORKDIR}/results/artifacts/baseline/cli|${WORKDIR}/results/artifacts/baseline/conformance|${ROOT}|podman-options"
|
|
assert_contains "${WORKDIR}/calls" "candidate|2|${WORKDIR}/results/artifacts/candidate/gateway|${WORKDIR}/results/artifacts/candidate/cli|${WORKDIR}/results/artifacts/candidate/conformance|${ROOT}|podman-options"
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"scenario":"podman-options"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"command_class":"podman_options"'
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"normalized_result":"baseline.normalized.json"'
|
|
assert_contains "${WORKDIR}/results/baseline.normalized.json" '"stable":true'
|
|
assert_contains "${WORKDIR}/results/baseline.normalized.json" '"pids_limit":2048'
|
|
assert_contains "${WORKDIR}/results/candidate.normalized.json" '"pids_limit":31'
|
|
assert_not_contains "${WORKDIR}/results/baseline.json" '"scenarios"'
|
|
assert_contains "${WORKDIR}/results/baseline.log" '"scenarios"'
|
|
assert_not_contains "${WORKDIR}/results/baseline.json" 'raw output'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"scenario":"podman-options"'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"parity":false'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"classification":"intentional_change"'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"intentional_change_id":"podman-pid-limit-restored"'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"accepted":true'
|
|
|
|
set +e
|
|
OPENSHELL_PARITY_TEST_SEMANTIC_DRIFT=1 run_harness --scenario podman-options >"${WORKDIR}/drift.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 1
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"accepted":false'
|
|
|
|
# Refuse an existing output path instead of consuming stale evidence from it.
|
|
[ -s "${WORKDIR}/results/candidate.normalized.json" ] || fail 'stale result fixture is missing'
|
|
set +e
|
|
OPENSHELL_PARITY_TEST_KEEP_RESULTS_DIR=1 run_harness --scenario podman-options >"${WORKDIR}/stale-results.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 2
|
|
assert_contains "${WORKDIR}/stale-results.out" 'parity results directory already exists'
|
|
|
|
# A fresh run whose wrapper emits no candidate result must fail.
|
|
set +e
|
|
OPENSHELL_PARITY_TEST_SKIP_RESULT=candidate run_harness --scenario podman-options >"${WORKDIR}/missing-result.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 1
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"accepted":false'
|
|
|
|
set +e
|
|
OPENSHELL_PARITY_FAIL_VARIANT=both run_harness >"${WORKDIR}/failure.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 1
|
|
assert_contains "${WORKDIR}/results/baseline.json" '"success":false'
|
|
assert_contains "${WORKDIR}/results/candidate.json" '"success":false'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"parity":false'
|
|
[ "$(wc -l <"${WORKDIR}/calls")" -eq 14 ] || fail 'candidate did not run after baseline failure'
|
|
|
|
set +e
|
|
OPENSHELL_PARITY_TEST_MUTATE_ARTIFACT=candidate run_harness >"${WORKDIR}/mutation.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 1
|
|
assert_contains "${WORKDIR}/mutation.out" 'candidate gateway changed after it was staged for execution'
|
|
assert_contains "${WORKDIR}/results/candidate.json" '"success":false'
|
|
assert_contains "${WORKDIR}/results/comparison.json" '"classification":"regression"'
|
|
|
|
set +e
|
|
bash "${ROOT}/e2e/parity/run.sh" --driver docker >"${WORKDIR}/driver.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 2
|
|
assert_contains "${WORKDIR}/driver.out" 'only --driver podman is supported'
|
|
|
|
set +e
|
|
bash "${ROOT}/e2e/parity/run.sh" --driver >"${WORKDIR}/option.out" 2>&1
|
|
status=$?
|
|
set -e
|
|
assert_status "${status}" 2
|
|
assert_contains "${WORKDIR}/option.out" '--driver requires a value'
|
|
|
|
echo 'e2e parity deterministic tests passed.'
|