trending/OpenShell
1
0
Fork 0
mirror of https://github.com/NVIDIA/OpenShell.git synced 2026-10-04 00:23:53 +08:00
Code Issues Packages Projects Releases Wiki Activity
Files
vm-runtime
OpenShell/tasks/test.toml
T
Taylor MutchDrew NewberrydiveshDivesh Chowdary
 0a770d9173 feat(kubernetes): support HA gateway rebalancing (#1868) 
* feat(kubernetes): support HA gateway rebalancing

Signed-off-by: Drew Newberry <anewberry@nvidia.com>

* perf(server): cache peer connections, tokens, and owner lookups

Every forwarded relay rebuilt its setup from scratch: an owner lookup, a
blocking read of the peer token, a TLS connect to the owning replica, and
a TokenReview plus Pod GET on the receiving side. Sandbox service routing
does this per HTTP request, so the apiserver calls scaled with traffic.

Cache all of it on ServerState:

- peer channels pooled per endpoint, so relays multiplex over one
  connection instead of redialing
- peer tokens keyed by SHA-256, expiring at min(ttl, token exp) so a hit
  cannot accept an expired token
- owner records for 3s against a 45s ownership TTL, still freshness
  checked before use

Entries are evicted when a relay fails. Also raise HTTP/2
max_concurrent_streams to 1024, since pooling funnels every relay between
two replicas onto one connection and hyper's default of 200 sits below
the 256 pending-relay budget.

Signed-off-by: divesh <dgude@nvidia.com>

* perf(server): pool upstream connections for sandbox services

Each HTTP request to a sandbox service opened its own supervisor relay,
paying a new TCP connection and HTTP/1 handshake every time. Worse, it
counted against the 32 in-flight relay cap, so a service handling more
than 32 concurrent requests failed outright.

Pool idle upstreams per endpoint and port, up to 8 each for 15s. Reuse is
safe because the pool only returns a connection hyper reports as ready,
and HTTP/1 cannot start a request until the previous body has drained.
Upgrades are never pooled since they take the connection over, and a
failed send evicts that endpoint. Pruning is bounded per key, with the
full sweep limited to once per 30s.

Signed-off-by: divesh <dgude@nvidia.com>

* fix(server): address HA gateway review findings (#3449)

- Let a gateway own supervisor sessions without a peer endpoint. Requiring
  one whenever the store is PostgreSQL broke every single-instance
  PostgreSQL deployment, because no sandbox supervisor could connect.
  A cross-replica request to an owner that advertises no endpoint now fails
  immediately naming the cause, instead of retrying until the wait timeout.
- Close a supervisor session on heartbeat only when another replica owns it,
  or after renewals fail for the ownership TTL. A database error no longer
  drops every session heartbeating during an outage.
- Clamp owner record ages at zero so a skewed or corrupt stored timestamp
  cannot produce a negative age.
- Bound the cross-object advisory lock with a lock timeout, so a stuck holder
  fails instead of blocking every mutation in the fleet.
- Refuse to start when a peer endpoint is configured on a multi-replica
  backend but peer authentication is unavailable, and warn when a
  multi-replica backend has no peer endpoint at all.
- Reject a plaintext peer endpoint when the gateway serves TLS.
- Skip the sandbox watch poller on single-replica backends, where the local
  update bus already sees every write.
- Rate-limit the peer owner cache sweep so an insert no longer scans the
  whole map under the lock.
- Retry GET and HEAD on a pooled upstream the sandbox closed, instead of
  returning 502, and drop an emptied endpoint from the pool right away.
- Document the gateway peer environment variables and the post-rollout
  ownership skew operators should expect.

Signed-off-by: divesh <dgude@nvidia.com>

* fix(server): harden HA supervisor ownership

Signed-off-by: Drew Newberry <anewberry@nvidia.com>

---------

Signed-off-by: Drew Newberry <anewberry@nvidia.com>
Signed-off-by: divesh <dgude@nvidia.com>
Co-authored-by: Drew Newberry <anewberry@nvidia.com>
Co-authored-by: divesh <dgude@nvidia.com>
Co-authored-by: Divesh Chowdary <47188680+FrostGod@users.noreply.github.com>
2026-09-21 21:22:12 +00:00

286 lines
15 KiB
TOML
Raw Permalink Blame History

# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Test tasks (Rust + Python + TypeScript SDK)
[test]
description = "Run all tests (Rust + Python + TypeScript SDK)"
depends = [
"test:rust",
"test:python",
"sdk:ts:test",
"test:sbom",
"test:install-sh",
"test:build-env",
"test:gateway-pull-policy",
"test:gateway-config",
"test:e2e-parity",
"test:packaging-assets",
"test:codex-security-release-range",
"test:docs-website",
]
["test:docs-website"]
description = "Test the docs-website sync script"
# --no-project skips installing the OpenShell package; --with supplies the test
# dependencies and the script's runtime dependency, which live outside the project env.
run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/sync_docs_website_test.py"
["test:sbom"]
description = "Run SBOM tooling tests"
run = "uv run --no-project --with pytest pytest -o \"python_files=*_test.py\" deploy/sbom/"
hide = true
["test:install-sh"]
description = "Run focused install.sh shell tests"
run = "tasks/scripts/test-install-sh.sh"
run_windows = "echo Skipping test:install-sh: Linux glibc installer tests do not apply on Windows."
hide = true
["test:build-env"]
description = "Run build-env.sh helper shell tests"
run = "tasks/scripts/test-build-env.sh"
run_windows = "echo Skipping test:build-env: the Unix build-env.sh helper does not apply on Windows."
hide = true
["test:gateway-pull-policy"]
description = "Test development gateway image pull-policy normalization"
run = "tasks/scripts/test-gateway-pull-policy.sh"
run_windows = "echo Skipping test:gateway-pull-policy: Unix gateway scripts do not apply on Windows."
hide = true
["test:packaging-assets"]
description = "Run static packaging asset tests"
run = "tasks/scripts/test-packaging-assets.sh"
run_windows = "echo Skipping test:packaging-assets: Linux service and RPM assets do not apply on Windows."
hide = true
["test:codex-security-release-range"]
description = "Test Codex Security release-range resolution"
run = "uv run --no-project --with pytest pytest -o \"python_files=*_test.py\" tasks/scripts/codex_security_range_test.py"
hide = true
[e2e]
description = "Run all end-to-end tests (Rust + Python + MCP)"
depends = ["e2e:rust", "e2e:python", "e2e:mcp"]
["e2e:test"]
description = "Build the current checkout and run a named host or Nix test-guest E2E suite"
run = "e2e/run.sh"
["e2e:gpu"]
description = "Run Docker GPU end-to-end tests"
depends = ["e2e:docker:gpu"]
["e2e:workloads:build"]
description = "Build local GPU workload test images and manifest"
run = "bash tasks/scripts/e2e-gpu-build-images.sh"
["test:rust"]
description = "Run Rust tests"
depends = ["rust:lockfiles:check"]
env = { OPENSHELL_TELEMETRY_ENABLED = "false" }
run = [
# Run the workspace once without openshell-server so we can run that crate
# with test-only helpers enabled.
"cargo test --workspace --exclude openshell-server",
"cargo test -p openshell-server --features test-support",
"cargo nextest run --config-file .config/nextest.toml --manifest-path examples/supervisor-middleware-content-guard/Cargo.toml",
]
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/windows-msvc.ps1 test-precommit native"
hide = true
["test:python"]
description = "Run Python tests"
depends = ["python:proto"]
env = { UV_NO_SYNC = "1" }
run = "uv run pytest python/"
hide = true
["e2e:rust"]
description = "Run Rust CLI e2e tests against a Docker-backed gateway"
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh"
["e2e:conformance:build"]
description = "Build the standalone CLI conformance binary"
run = "if [ -z \"${OPENSHELL_CONFORMANCE_BIN:-}\" ]; then cargo build -p openshell-conformance-cli; fi"
hide = true
["e2e:cli-conformance"]
description = "Build and run the standalone CLI conformance suite against the configured gateway"
depends = ["e2e:conformance:build"]
run = [
"if [ -z \"${OPENSHELL_BIN:-}\" ]; then cargo build -p openshell-cli; fi",
"\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" run --openshell-bin \"${OPENSHELL_BIN:-$PWD/target/debug/openshell}\"",
]
["e2e:websocket-conformance"]
description = "Run focused WebSocket conformance e2e tests against a Docker-backed gateway"
run = [
"e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test websocket_conformance",
]
["e2e:mcp"]
description = "Run MCP conformance e2e scenarios against one Docker-backed gateway (static defaults for spec 2025-11-25; set OPENSHELL_MCP_CONFORMANCE_SCENARIOS for a focused subset)"
run = "bash e2e/mcp-conformance.sh"
["e2e:nodejs"]
description = "Alias for e2e:mcp"
depends = ["e2e:mcp"]
["e2e:python"]
description = "Run Python e2e tests against a Docker-backed gateway (E2E_PARALLEL=N or 'auto'; default 5)"
depends = ["python:proto"]
env = { UV_NO_SYNC = "1", PYTHONPATH = "python" }
run = "e2e/with-docker-gateway.sh uv run pytest -o python_files='test_*.py *_test.py' -m 'not gpu' -n ${E2E_PARALLEL:-5} e2e/python"
["e2e:podman"]
description = "Run Rust CLI e2e tests against a Podman-backed gateway"
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh"
["e2e:oidc-pkce"]
description = "Run Linux browser PKCE and RBAC e2e tests against Keycloak and a Podman gateway"
run = [
"CONTAINER_RUNTIME=podman e2e/with-keycloak.sh env OPENSHELL_E2E_OIDC_GATEWAY=1 e2e/with-podman-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-oidc-pkce --test oidc_pkce",
]
["e2e:oidc-pkce:docker"]
description = "Run Linux browser PKCE and RBAC e2e tests against Keycloak and a Docker gateway"
run = [
"CONTAINER_RUNTIME=docker e2e/with-keycloak.sh env OPENSHELL_E2E_OIDC_GATEWAY=1 e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-oidc-pkce --test oidc_pkce",
]
["e2e:oidc-python:docker"]
description = "Run Python OIDC and workspace authorization e2e tests against Keycloak and a Docker gateway"
depends = ["python:proto"]
env = { UV_NO_SYNC = "1", PYTHONPATH = "python" }
run = [
"CONTAINER_RUNTIME=docker e2e/with-keycloak.sh env OPENSHELL_E2E_OIDC_GATEWAY=1 e2e/with-docker-gateway.sh uv run pytest -m 'not gpu' e2e/python/oidc",
]
["e2e:podman:gpu"]
description = "Run GPU e2e against a standalone gateway with the Podman compute driver"
env = { OPENSHELL_E2E_PODMAN_GPU = "1", OPENSHELL_E2E_PODMAN_TEST = "gpu", OPENSHELL_E2E_PODMAN_FEATURES = "e2e-podman-gpu" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh"
["e2e:kubernetes"]
description = "Run Rust CLI e2e tests against an OpenShell gateway deployed on Kubernetes via Helm (set OPENSHELL_E2E_KUBE_CONTEXT to reuse a cluster; otherwise creates a local k3d cluster when k3d is installed; set OPENSHELL_E2E_KUBE_TEST=<name> to scope to one test)"
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
["e2e:kubernetes:v1alpha1"]
description = "Run Kubernetes e2e against Agent Sandbox v1alpha1"
env = { AGENT_SANDBOX_VERSION = "v0.4.6" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
["e2e:kubernetes:agent-sandbox-versions"]
description = "Run Kubernetes e2e against Agent Sandbox v1beta1 and v1alpha1"
depends = ["e2e:conformance:build"]
run = [
"OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh",
"OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh",
]
["e2e:kubernetes:isolation"]
description = "Run Kubernetes e2e with the workload network fence and separate supervisor"
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
["e2e:kubernetes:db"]
description = "Run Kubernetes e2e with all database backend scenarios (SQLite and external PostgreSQL with existingSecret)"
env = { OPENSHELL_E2E_KUBE_DB_SCENARIOS = "1" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
["e2e:kubernetes:ha-rebalancing"]
description = "Run the Kubernetes HA rebalancing suite through Envoy against two gateway replicas and external PostgreSQL"
env = { OPENSHELL_E2E_KUBE_EXTERNAL_POSTGRES_SECRET = "openshell-ha-pg", OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-high-availability.yaml", OPENSHELL_E2E_KUBE_TEST = "kubernetes_ha_rebalancing", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-host-gateway,e2e-kubernetes,e2e-kubernetes-ha", OPENSHELL_E2E_KUBE_USE_ENVOY = "1" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
["e2e:kubernetes:credential-drivers"]
description = "Run Kubernetes e2e for provider credential storage backed by Kubernetes Secrets and Vault"
env = { OPENSHELL_E2E_CREDENTIAL_DRIVERS = "1", OPENSHELL_E2E_KUBE_TEST = "credential_drivers", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-credential-drivers" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
["e2e:kubernetes:workspace-managed"]
description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)"
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
["e2e:kubernetes:workspace-operator"]
description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)"
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
["e2e:vm"]
description = "Start openshell-gateway with the VM compute driver and run VM e2e tests"
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh"
["e2e:gateway:no-compute-drivers"]
description = "Build and launch-check openshell-gateway without compiled compute drivers"
run = "bash e2e/no-compute-driver-gateway.sh"
["e2e:docker:external-driver"]
description = "Run Docker conformance with a driver-free gateway and external Docker driver binary"
env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_DOCKER_FEATURES = "" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh"
["e2e:podman:external-driver"]
description = "Run Podman conformance with a driver-free gateway and external Podman driver binary"
env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_PODMAN_FEATURES = "" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-podman.sh"
["e2e:vm:external-driver"]
description = "Run VM E2E with a driver-free gateway and external VM driver binary"
env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh"
["e2e:kubernetes:external-driver"]
description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver"
env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
["e2e:docker"]
description = "Run Docker conformance and Rust e2e tests against a standalone gateway"
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh"
["e2e:mechanistic-smoke"]
description = "Run mechanistic L4 smoke against a Docker-backed gateway"
run = [
"cargo build -p openshell-cli",
"e2e/with-docker-gateway.sh bash -lc 'target/debug/openshell settings set --global --key agent_policy_proposals_enabled --value true --yes && OPENSHELL_BIN=$PWD/target/debug/openshell bash e2e/policy-advisor/mechanistic-smoke.sh'",
]
["e2e:mechanistic-existing-endpoint"]
description = "Run #2821 existing inspected-endpoint auto-approval regression"
run = [
"cargo build -p openshell-cli",
"e2e/with-docker-gateway.sh bash -lc 'target/debug/openshell settings set --global --key agent_policy_proposals_enabled --value true --yes && OPENSHELL_BIN=$PWD/target/debug/openshell bash e2e/policy-advisor/existing-endpoint-auto-approve.sh'",
]
["e2e:docker:gpu"]
description = "Run GPU e2e against a standalone gateway with the Docker compute driver"
env = { OPENSHELL_E2E_DOCKER_GPU = "1", OPENSHELL_E2E_DOCKER_TEST = "gpu", OPENSHELL_E2E_DOCKER_FEATURES = "e2e-docker-gpu" }
depends = ["e2e:conformance:build"]
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-docker.sh"
["test:gateway-config"]
description = "Test generated local gateway TOML without starting a runtime"
run = "bash tasks/scripts/test-gateway-config.sh"
run_windows = "echo Skipping test:gateway-config: Unix gateway scripts do not apply on Windows."
hide = true
Reference in New Issue View Git Blame Copy Permalink