mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-19 05:50:41 +08:00
0a770d9173
* feat(kubernetes): support HA gateway rebalancing Signed-off-by: Drew Newberry <anewberry@nvidia.com> * perf(server): cache peer connections, tokens, and owner lookups Every forwarded relay rebuilt its setup from scratch: an owner lookup, a blocking read of the peer token, a TLS connect to the owning replica, and a TokenReview plus Pod GET on the receiving side. Sandbox service routing does this per HTTP request, so the apiserver calls scaled with traffic. Cache all of it on ServerState: - peer channels pooled per endpoint, so relays multiplex over one connection instead of redialing - peer tokens keyed by SHA-256, expiring at min(ttl, token exp) so a hit cannot accept an expired token - owner records for 3s against a 45s ownership TTL, still freshness checked before use Entries are evicted when a relay fails. Also raise HTTP/2 max_concurrent_streams to 1024, since pooling funnels every relay between two replicas onto one connection and hyper's default of 200 sits below the 256 pending-relay budget. Signed-off-by: divesh <dgude@nvidia.com> * perf(server): pool upstream connections for sandbox services Each HTTP request to a sandbox service opened its own supervisor relay, paying a new TCP connection and HTTP/1 handshake every time. Worse, it counted against the 32 in-flight relay cap, so a service handling more than 32 concurrent requests failed outright. Pool idle upstreams per endpoint and port, up to 8 each for 15s. Reuse is safe because the pool only returns a connection hyper reports as ready, and HTTP/1 cannot start a request until the previous body has drained. Upgrades are never pooled since they take the connection over, and a failed send evicts that endpoint. Pruning is bounded per key, with the full sweep limited to once per 30s. Signed-off-by: divesh <dgude@nvidia.com> * fix(server): address HA gateway review findings (#3449) - Let a gateway own supervisor sessions without a peer endpoint. Requiring one whenever the store is PostgreSQL broke every single-instance PostgreSQL deployment, because no sandbox supervisor could connect. A cross-replica request to an owner that advertises no endpoint now fails immediately naming the cause, instead of retrying until the wait timeout. - Close a supervisor session on heartbeat only when another replica owns it, or after renewals fail for the ownership TTL. A database error no longer drops every session heartbeating during an outage. - Clamp owner record ages at zero so a skewed or corrupt stored timestamp cannot produce a negative age. - Bound the cross-object advisory lock with a lock timeout, so a stuck holder fails instead of blocking every mutation in the fleet. - Refuse to start when a peer endpoint is configured on a multi-replica backend but peer authentication is unavailable, and warn when a multi-replica backend has no peer endpoint at all. - Reject a plaintext peer endpoint when the gateway serves TLS. - Skip the sandbox watch poller on single-replica backends, where the local update bus already sees every write. - Rate-limit the peer owner cache sweep so an insert no longer scans the whole map under the lock. - Retry GET and HEAD on a pooled upstream the sandbox closed, instead of returning 502, and drop an emptied endpoint from the pool right away. - Document the gateway peer environment variables and the post-rollout ownership skew operators should expect. Signed-off-by: divesh <dgude@nvidia.com> * fix(server): harden HA supervisor ownership Signed-off-by: Drew Newberry <anewberry@nvidia.com> --------- Signed-off-by: Drew Newberry <anewberry@nvidia.com> Signed-off-by: divesh <dgude@nvidia.com> Co-authored-by: Drew Newberry <anewberry@nvidia.com> Co-authored-by: divesh <dgude@nvidia.com> Co-authored-by: Divesh Chowdary <47188680+FrostGod@users.noreply.github.com>
255 lines
6.5 KiB
TOML
255 lines
6.5 KiB
TOML
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
# Standalone crate — the empty [workspace] table prevents Cargo from
|
|
# treating this as part of the root workspace, avoiding Dockerfile
|
|
# and Cargo.lock coupling.
|
|
[workspace]
|
|
|
|
[package]
|
|
name = "openshell-e2e"
|
|
description = "End-to-end tests for the OpenShell CLI"
|
|
version = "0.1.0"
|
|
edition = "2024"
|
|
rust-version = "1.90"
|
|
license = "Apache-2.0"
|
|
publish = false
|
|
|
|
[features]
|
|
# Selects the common E2E suite.
|
|
e2e = []
|
|
# Selects tests that rely on `host.openshell.internal` (the sandbox's stable
|
|
# alias to the host running test fixtures). docker, podman, and vm wire the
|
|
# alias unconditionally; the kube driver only does so when the chart's
|
|
# `server.hostGatewayIP` is set, so `e2e-kubernetes` does NOT imply this and
|
|
# the helm wrapper opts in explicitly when it has resolved an IP.
|
|
e2e-host-gateway = ["e2e"]
|
|
e2e-local-container-driver = ["e2e"]
|
|
e2e-docker = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"]
|
|
e2e-gpu = ["e2e"]
|
|
e2e-docker-gpu = ["e2e-docker", "e2e-gpu"]
|
|
e2e-kubernetes = ["e2e"]
|
|
e2e-kubernetes-ha = ["e2e-kubernetes"]
|
|
e2e-kubernetes-credential-drivers = ["e2e-kubernetes"]
|
|
e2e-kubernetes-workspace-managed = ["e2e-kubernetes"]
|
|
e2e-kubernetes-workspace-operator = ["e2e-kubernetes"]
|
|
e2e-podman = ["e2e", "e2e-host-gateway", "e2e-local-container-driver"]
|
|
e2e-podman-gpu = ["e2e-podman", "e2e-gpu"]
|
|
e2e-oidc-pkce = []
|
|
e2e-vm = ["e2e", "e2e-host-gateway"]
|
|
|
|
[[test]]
|
|
name = "policy_activation"
|
|
path = "tests/policy_activation.rs"
|
|
required-features = ["e2e-docker"]
|
|
|
|
[[test]]
|
|
name = "oidc_pkce"
|
|
path = "tests/oidc_pkce.rs"
|
|
required-features = ["e2e-oidc-pkce"]
|
|
|
|
[[test]]
|
|
name = "vm_overlay"
|
|
path = "tests/vm_overlay.rs"
|
|
required-features = ["e2e-vm"]
|
|
|
|
[[test]]
|
|
name = "custom_image"
|
|
path = "tests/custom_image.rs"
|
|
required-features = ["e2e-docker"]
|
|
|
|
[[test]]
|
|
name = "rootfs_tar"
|
|
path = "tests/rootfs_tar.rs"
|
|
required-features = ["e2e-vm"]
|
|
|
|
[[test]]
|
|
name = "docker_preflight"
|
|
path = "tests/docker_preflight.rs"
|
|
required-features = ["e2e-docker"]
|
|
|
|
[[test]]
|
|
name = "driver_config_volume"
|
|
path = "tests/driver_config_volume.rs"
|
|
required-features = ["e2e-local-container-driver"]
|
|
|
|
[[test]]
|
|
name = "gateway_start"
|
|
path = "tests/gateway_start.rs"
|
|
required-features = ["e2e-docker"]
|
|
|
|
[[test]]
|
|
name = "local_driver_token_restart"
|
|
path = "tests/local_driver_token_restart.rs"
|
|
required-features = ["e2e"]
|
|
|
|
[[test]]
|
|
name = "podman_gateway_start"
|
|
path = "tests/podman_gateway_start.rs"
|
|
required-features = ["e2e-podman"]
|
|
|
|
[[test]]
|
|
name = "podman_corporate_proxy"
|
|
path = "tests/podman_corporate_proxy.rs"
|
|
required-features = ["e2e-podman"]
|
|
|
|
[[test]]
|
|
name = "podman_oci_identity"
|
|
path = "tests/podman_oci_identity.rs"
|
|
required-features = ["e2e-podman"]
|
|
|
|
[[test]]
|
|
name = "podman_userns"
|
|
path = "tests/podman_userns.rs"
|
|
required-features = ["e2e-podman"]
|
|
|
|
[[test]]
|
|
name = "provider_refresh_handles"
|
|
path = "tests/provider_refresh_handles.rs"
|
|
required-features = ["e2e-podman"]
|
|
|
|
[[test]]
|
|
name = "provider_readiness"
|
|
path = "tests/provider_readiness.rs"
|
|
required-features = ["e2e-docker"]
|
|
|
|
[[test]]
|
|
name = "vm_gateway_start"
|
|
path = "tests/vm_gateway_start.rs"
|
|
required-features = ["e2e-vm"]
|
|
|
|
[[test]]
|
|
name = "vm_corporate_proxy"
|
|
path = "tests/vm_corporate_proxy.rs"
|
|
required-features = ["e2e-vm"]
|
|
|
|
[[test]]
|
|
name = "provider_token_exchange"
|
|
path = "tests/provider_token_exchange.rs"
|
|
required-features = ["e2e-podman"]
|
|
|
|
[[test]]
|
|
name = "kubernetes_ha_rebalancing"
|
|
path = "tests/kubernetes_ha_rebalancing.rs"
|
|
required-features = ["e2e-kubernetes-ha"]
|
|
|
|
[[test]]
|
|
name = "kubernetes_corporate_proxy"
|
|
path = "tests/kubernetes_corporate_proxy.rs"
|
|
required-features = ["e2e-kubernetes"]
|
|
|
|
[[test]]
|
|
name = "credential_drivers"
|
|
path = "tests/credential_drivers.rs"
|
|
required-features = ["e2e-kubernetes-credential-drivers"]
|
|
|
|
[[test]]
|
|
name = "websocket_conformance"
|
|
path = "tests/websocket_conformance.rs"
|
|
required-features = ["e2e-host-gateway"]
|
|
|
|
[[test]]
|
|
name = "credential_gating"
|
|
path = "tests/credential_gating.rs"
|
|
required-features = ["e2e-host-gateway"]
|
|
|
|
[[test]]
|
|
name = "user_namespaces"
|
|
path = "tests/user_namespaces.rs"
|
|
required-features = ["e2e-kubernetes"]
|
|
|
|
[[test]]
|
|
name = "host_gateway_alias"
|
|
path = "tests/host_gateway_alias.rs"
|
|
required-features = ["e2e-host-gateway"]
|
|
|
|
[[test]]
|
|
name = "forward_proxy_l7_bypass"
|
|
path = "tests/forward_proxy_l7_bypass.rs"
|
|
required-features = ["e2e-host-gateway"]
|
|
|
|
[[test]]
|
|
name = "forward_proxy_graphql_l7"
|
|
path = "tests/forward_proxy_graphql_l7.rs"
|
|
required-features = ["e2e-host-gateway"]
|
|
|
|
[[test]]
|
|
name = "forward_proxy_jsonrpc_l7"
|
|
path = "tests/forward_proxy_jsonrpc_l7.rs"
|
|
required-features = ["e2e-host-gateway"]
|
|
|
|
[[test]]
|
|
name = "workspace_lifecycle"
|
|
path = "tests/workspace_lifecycle.rs"
|
|
required-features = ["e2e"]
|
|
|
|
[[test]]
|
|
name = "sandbox_templates"
|
|
path = "tests/sandbox_templates.rs"
|
|
required-features = ["e2e"]
|
|
|
|
[[test]]
|
|
name = "proxy_egress_pipeline"
|
|
path = "tests/proxy_egress_pipeline.rs"
|
|
required-features = ["e2e-host-gateway"]
|
|
|
|
[[test]]
|
|
name = "workspace_namespace_managed"
|
|
path = "tests/workspace_namespace_managed.rs"
|
|
required-features = ["e2e-kubernetes-workspace-managed"]
|
|
|
|
[[test]]
|
|
name = "workspace_namespace_operator"
|
|
path = "tests/workspace_namespace_operator.rs"
|
|
required-features = ["e2e-kubernetes-workspace-operator"]
|
|
|
|
[[test]]
|
|
name = "gpu"
|
|
path = "tests/gpu.rs"
|
|
required-features = ["e2e-gpu"]
|
|
|
|
[dependencies]
|
|
base64 = "0.22"
|
|
bollard = "0.20"
|
|
bytes = "1"
|
|
futures-util = "0.3"
|
|
http-body-util = "0.1"
|
|
hyper = { version = "1", features = ["client", "http1"] }
|
|
hyper-util = { version = "0.1", features = ["tokio"] }
|
|
jsonwebtoken = { version = "10", features = ["aws_lc_rs"] }
|
|
prost = "0.14"
|
|
tokio = { version = "1.43", features = ["full"] }
|
|
tokio-stream = { version = "0.1", features = ["net"] }
|
|
tempfile = "3"
|
|
sha1 = "0.10"
|
|
sha2 = "0.10"
|
|
hex = "0.4"
|
|
rand = "0.9"
|
|
serde = { version = "1", features = ["derive"] }
|
|
serde_json = "1"
|
|
serde_yml = { package = "noyalib", version = "0.0.28", default-features = false, features = ["std", "compat-serde-yaml"] }
|
|
tonic = { version = "0.14", features = ["transport"] }
|
|
tonic-prost = "0.14"
|
|
tower = "0.5"
|
|
url = "2"
|
|
nix = { version = "0.29", features = ["user"] }
|
|
|
|
[dev-dependencies]
|
|
serial_test = "3"
|
|
|
|
[lints.rust]
|
|
unsafe_code = "warn"
|
|
rust_2018_idioms = { level = "warn", priority = -1 }
|
|
|
|
[lints.clippy]
|
|
all = { level = "warn", priority = -1 }
|
|
pedantic = { level = "warn", priority = -1 }
|
|
module_name_repetitions = "allow"
|
|
must_use_candidate = "allow"
|
|
missing_errors_doc = "allow"
|
|
missing_panics_doc = "allow"
|
|
[[test]]
|
|
name = "live_internet_traffic_perf"
|
|
path = "tests/live_internet_traffic_perf.rs"
|
|
required-features = ["e2e-host-gateway"]
|