feat(sdk): add extension auth for middleware E2E

Signed-off-by: Piotr Mlocek <pmlocek@nvidia.com>
This commit is contained in:
Piotr Mlocek
2026-08-21 09:27:39 -07:00
parent de4c1fecf5
commit bd7016ff84
16 changed files with 3826 additions and 0 deletions
+3
View File
@@ -62,6 +62,9 @@ jobs:
- suite: rust-docker-external-driver
cmd: "env -u OPENSHELL_GATEWAY_BIN mise run --no-deps --skip-deps e2e:docker:external-driver"
apt_packages: "openssh-client"
- suite: supervisor-middleware
cmd: "mise run --no-deps --skip-deps e2e:supervisor-middleware"
apt_packages: "openssh-client"
- suite: mcp
cmd: "mise run --no-deps --skip-deps e2e:mcp"
apt_packages: ""
Generated
+1
View File
@@ -4140,6 +4140,7 @@ dependencies = [
"futures",
"hyper 1.9.0",
"hyper-util",
"jsonwebtoken",
"miette",
"oauth2",
"openshell-core",
+4
View File
@@ -87,6 +87,10 @@ rotates their in-memory slots without rebuilding clients, and publishes the
public verification key at `/.well-known/jwks.json` alongside OIDC-shaped
discovery metadata at `/.well-known/openid-configuration`. HTTPS extensions can
pin an operator-provided CA while retaining endpoint-hostname verification.
Rust extension services use the opt-in `openshell-sdk::extension` surface to
verify these credentials from operator-provisioned Ed25519 public keys or JWKS
documents and normalize gateway and supervisor caller identities. Key refresh
remains a service deployment concern.
Extension credentials reuse the sandbox signing key and are separated from
sandbox-to-gateway admission tokens by exact audience and by an explicit
+9
View File
@@ -10,17 +10,26 @@ rust-version.workspace = true
license.workspace = true
repository.workspace = true
[features]
default = []
extension = [
"dep:jsonwebtoken",
"dep:serde_json",
]
[dependencies]
openshell-core = { path = "../openshell-core" }
async-trait = "0.1"
futures = { workspace = true }
hyper = { workspace = true }
hyper-util = { workspace = true }
jsonwebtoken = { workspace = true, optional = true }
miette = { workspace = true }
oauth2 = { version = "5", default-features = false, features = ["reqwest"] }
reqwest = { workspace = true }
rustls = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true, optional = true }
thiserror = { workspace = true }
tokio = { workspace = true }
tokio-rustls = { workspace = true }
+15
View File
@@ -49,6 +49,20 @@ mTLS (client certificates) is not supported.
SDK-shaped enums rather than raw proto integers. Failures map to a typed
`SdkError` with a discriminable kind.
Enable the `extension` Cargo feature when implementing an operator-run
OpenShell extension service:
```toml
openshell-sdk = { version = "...", features = ["extension"] }
```
The `extension` module verifies gateway-minted EdDSA extension JWTs from a
trusted public key or JWKS document. `GatewayJwtAuthenticator` pins the token
type, algorithm, issuer, audience, lifetime, and caller identity shape, then
returns an `AuthenticatedCaller` that service handlers can authorize by caller
kind. This feature is opt-in so gateway-only SDK consumers do not compile the
extension verification dependencies.
## Modules
| Module | Purpose |
@@ -60,6 +74,7 @@ SDK-shaped enums rather than raw proto integers. Failures map to a typed
| `oidc` | OIDC token handling at the transport layer. |
| `refresh` | `Refresh` trait and single-flight refresh coalescing. |
| `edge_tunnel` | Cloudflare Access tunnel dialer. |
| `extension` | Opt-in service-side authentication and identity types for operator-run extensions. |
| `error` | `SdkError` taxonomy. |
| `types` | Curated request/response types and proto conversions. |
| `raw` | Escape hatch re-exporting the generated tonic clients. |
+13
View File
@@ -0,0 +1,13 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! APIs for implementing operator-run `OpenShell` extension services.
//!
//! Enable the `extension` Cargo feature to verify gateway-minted extension
//! credentials and authorize callers without depending on gateway internals.
mod verification;
pub use verification::{
AuthenticatedCaller, ExtensionCallerKind, GatewayJwtAuthenticator, VerificationError,
};
@@ -0,0 +1,409 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
use std::collections::HashMap;
use std::time::Duration;
use jsonwebtoken::{
Algorithm, DecodingKey, Validation, decode, decode_header,
jwk::{AlgorithmParameters, EllipticCurve, JwkSet, PublicKeyUse},
};
use serde::{Deserialize, Serialize};
use tonic::{Request, Status};
const CLOCK_SKEW_SECS: i64 = 30;
const EXTENSION_JWT_TYP: &str = "openshell-ext+jwt";
const MAX_EXTENSION_TOKEN_TTL: Duration = Duration::from_secs(3_600);
/// `OpenShell` component authenticated as the caller of an extension service.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum ExtensionCallerKind {
Gateway,
Supervisor,
}
/// Private wire representation of gateway-minted extension claims.
///
/// Keeping the claims internal lets the SDK expose a stable, normalized caller
/// identity without making the gateway's token schema part of its public API.
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
struct ExtensionJwtClaims {
iss: String,
aud: String,
sub: String,
iat: i64,
exp: i64,
jti: String,
caller_kind: ExtensionCallerKind,
#[serde(skip_serializing_if = "Option::is_none")]
sandbox_id: Option<String>,
}
/// Verified identity presented by an `OpenShell` component to an extension.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct AuthenticatedCaller {
pub kind: ExtensionCallerKind,
pub subject: String,
pub sandbox_id: Option<String>,
}
/// Verifies gateway-minted JWTs presented to an extension service.
///
/// The public key or JWKS document must come from a trusted operator channel.
/// Parsing key material does not establish trust in it.
pub struct GatewayJwtAuthenticator {
keys: HashMap<String, DecodingKey>,
issuer: String,
audience: String,
}
impl std::fmt::Debug for GatewayJwtAuthenticator {
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
formatter
.debug_struct("GatewayJwtAuthenticator")
.field("key_ids", &self.keys.keys())
.field("issuer", &self.issuer)
.field("audience", &self.audience)
.finish()
}
}
impl GatewayJwtAuthenticator {
/// Construct a verifier from one trusted Ed25519 public key.
pub fn from_ed25519_pem(
public_key_pem: &[u8],
key_id: impl Into<String>,
issuer: impl Into<String>,
audience: impl Into<String>,
) -> Result<Self, VerificationError> {
let key_id = key_id.into();
if key_id.is_empty() {
return Err(VerificationError::MissingKeyId);
}
let key = DecodingKey::from_ed_pem(public_key_pem)
.map_err(VerificationError::InvalidPublicKey)?;
Self::new(HashMap::from([(key_id, key)]), issuer, audience)
}
/// Construct a verifier from a trusted JWKS document.
pub fn from_jwks(
jwks_json: &[u8],
issuer: impl Into<String>,
audience: impl Into<String>,
) -> Result<Self, VerificationError> {
let jwks: JwkSet = serde_json::from_slice(jwks_json)?;
let mut keys = HashMap::with_capacity(jwks.keys.len());
for jwk in jwks.keys {
let key_id = jwk
.common
.key_id
.clone()
.filter(|key_id| !key_id.is_empty())
.ok_or(VerificationError::MissingKeyId)?;
let supported = jwk.common.key_algorithm
== Some(jsonwebtoken::jwk::KeyAlgorithm::EdDSA)
&& jwk.common.public_key_use == Some(PublicKeyUse::Signature)
&& matches!(
&jwk.algorithm,
AlgorithmParameters::OctetKeyPair(parameters)
if parameters.curve == EllipticCurve::Ed25519
);
if !supported {
return Err(VerificationError::UnsupportedKey(key_id));
}
let key = DecodingKey::from_jwk(&jwk)
.map_err(|_| VerificationError::UnsupportedKey(key_id.clone()))?;
if keys.insert(key_id.clone(), key).is_some() {
return Err(VerificationError::DuplicateKeyId(key_id));
}
}
Self::new(keys, issuer, audience)
}
fn new(
keys: HashMap<String, DecodingKey>,
issuer: impl Into<String>,
audience: impl Into<String>,
) -> Result<Self, VerificationError> {
if keys.is_empty() {
return Err(VerificationError::EmptyKeySet);
}
let issuer = issuer.into();
let audience = audience.into();
if issuer.is_empty() || audience.is_empty() {
return Err(VerificationError::EmptyExpectedIdentity);
}
Ok(Self {
keys,
issuer,
audience,
})
}
/// Verify a bearer token and return its normalized caller identity.
pub fn authenticate(
&self,
bearer_token: &str,
) -> Result<AuthenticatedCaller, VerificationError> {
let header = decode_header(bearer_token).map_err(VerificationError::InvalidToken)?;
if header.typ.as_deref() != Some(EXTENSION_JWT_TYP) {
return Err(VerificationError::UnexpectedTokenType);
}
if header.alg != Algorithm::EdDSA {
return Err(VerificationError::UnexpectedAlgorithm);
}
let key_id = header
.kid
.as_deref()
.filter(|key_id| !key_id.is_empty())
.ok_or(VerificationError::MissingTokenKeyId)?;
let key = self
.keys
.get(key_id)
.ok_or_else(|| VerificationError::UnknownKeyId(key_id.to_string()))?;
let mut validation = Validation::new(Algorithm::EdDSA);
validation.algorithms = vec![Algorithm::EdDSA];
validation.leeway = u64::try_from(CLOCK_SKEW_SECS).unwrap_or_default();
validation.set_issuer(&[&self.issuer]);
validation.set_audience(&[&self.audience]);
validation.set_required_spec_claims(&[
"iss",
"aud",
"sub",
"iat",
"exp",
"jti",
"caller_kind",
]);
let claims = decode::<ExtensionJwtClaims>(bearer_token, key, &validation)
.map_err(VerificationError::InvalidToken)?
.claims;
validate_claim_shape(&claims)?;
Ok(AuthenticatedCaller {
kind: claims.caller_kind,
subject: claims.sub,
sandbox_id: claims.sandbox_id,
})
}
/// Extract and verify the bearer credential on a tonic request.
///
/// Detailed verification failures remain available through
/// [`Self::authenticate`]. This helper returns a stable gRPC error that
/// does not expose verifier internals to an untrusted caller.
pub fn authenticate_request<T>(
&self,
request: &Request<T>,
) -> Result<AuthenticatedCaller, Status> {
let authorization = request
.metadata()
.get("authorization")
.ok_or_else(|| Status::unauthenticated("missing extension authorization"))?
.to_str()
.map_err(|_| Status::unauthenticated("invalid extension authorization"))?;
let token = authorization
.strip_prefix("Bearer ")
.filter(|token| !token.is_empty())
.ok_or_else(|| Status::unauthenticated("expected an extension bearer token"))?;
self.authenticate(token)
.map_err(|_| Status::unauthenticated("invalid extension token"))
}
}
fn validate_claim_shape(claims: &ExtensionJwtClaims) -> Result<(), VerificationError> {
if claims.jti.is_empty() {
return Err(VerificationError::InvalidClaims("jti must not be empty"));
}
if claims.exp <= claims.iat {
return Err(VerificationError::InvalidClaims(
"exp must be later than iat",
));
}
let max_lifetime = i64::try_from(MAX_EXTENSION_TOKEN_TTL.as_secs()).unwrap_or(i64::MAX);
if claims.exp.saturating_sub(claims.iat) > max_lifetime {
return Err(VerificationError::InvalidClaims(
"token lifetime exceeds the extension maximum",
));
}
let now = i64::try_from(jsonwebtoken::get_current_timestamp()).unwrap_or(i64::MAX);
if claims.iat > now.saturating_add(CLOCK_SKEW_SECS) {
return Err(VerificationError::InvalidClaims(
"iat is later than the allowed clock skew",
));
}
match (claims.caller_kind, claims.sandbox_id.as_deref()) {
(ExtensionCallerKind::Gateway, None) if claims.sub == claims.iss => Ok(()),
(ExtensionCallerKind::Supervisor, Some(sandbox_id))
if !sandbox_id.is_empty()
&& claims.sub == format!("spiffe://openshell/sandbox/{sandbox_id}") =>
{
Ok(())
}
(ExtensionCallerKind::Gateway, _) => Err(VerificationError::InvalidClaims(
"gateway caller identity is inconsistent",
)),
(ExtensionCallerKind::Supervisor, _) => Err(VerificationError::InvalidClaims(
"supervisor caller identity is inconsistent",
)),
}
}
#[derive(Debug, thiserror::Error)]
pub enum VerificationError {
#[error("invalid JWKS document: {0}")]
InvalidJwks(#[from] serde_json::Error),
#[error("invalid Ed25519 public key: {0}")]
InvalidPublicKey(jsonwebtoken::errors::Error),
#[error("extension key is missing a key ID")]
MissingKeyId,
#[error("JWKS contains duplicate key ID '{0}'")]
DuplicateKeyId(String),
#[error("JWKS key '{0}' is not a supported Ed25519 signing key")]
UnsupportedKey(String),
#[error("extension key set is empty")]
EmptyKeySet,
#[error("expected issuer and audience must not be empty")]
EmptyExpectedIdentity,
#[error("token does not use the OpenShell extension JWT type")]
UnexpectedTokenType,
#[error("token does not use EdDSA")]
UnexpectedAlgorithm,
#[error("token header does not contain a key ID")]
MissingTokenKeyId,
#[error("token references unknown key ID '{0}'")]
UnknownKeyId(String),
#[error("token validation failed: {0}")]
InvalidToken(jsonwebtoken::errors::Error),
#[error("invalid extension claims: {0}")]
InvalidClaims(&'static str),
}
#[cfg(test)]
mod tests {
use jsonwebtoken::{EncodingKey, Header, encode};
use super::*;
const PRIVATE_KEY: &[u8] = b"-----BEGIN PRIVATE KEY-----\nMC4CAQAwBQYDK2VwBCIEIGrD/e7uKYqSY4twDEsRfMMuLSrODf14dpTiTK6K1YI0\n-----END PRIVATE KEY-----\n";
const PUBLIC_KEY: &[u8] = b"-----BEGIN PUBLIC KEY-----\nMCowBQYDK2VwAyEA2+Jj2UvNCvQiUPNYRgSi0cJSPiJI6Rs6D0UTeEpQVj8=\n-----END PUBLIC KEY-----\n";
const JWKS: &[u8] = br#"{"keys":[{"kty":"OKP","use":"sig","crv":"Ed25519","x":"2-Jj2UvNCvQiUPNYRgSi0cJSPiJI6Rs6D0UTeEpQVj8","kid":"test-key","alg":"EdDSA"}]}"#;
fn claims(kind: ExtensionCallerKind) -> ExtensionJwtClaims {
let now = i64::try_from(jsonwebtoken::get_current_timestamp()).unwrap();
let (subject, sandbox_id) = match kind {
ExtensionCallerKind::Gateway => ("openshell-gateway:test".into(), None),
ExtensionCallerKind::Supervisor => (
"spiffe://openshell/sandbox/sandbox-1".into(),
Some("sandbox-1".into()),
),
};
ExtensionJwtClaims {
iss: "openshell-gateway:test".into(),
aud: "urn:openshell:extension:middleware:test".into(),
sub: subject,
iat: now,
exp: now + 300,
jti: "unique".into(),
caller_kind: kind,
sandbox_id,
}
}
fn token(claims: &ExtensionJwtClaims, token_type: Option<&str>) -> String {
let mut header = Header::new(Algorithm::EdDSA);
header.kid = Some("test-key".into());
header.typ = token_type.map(str::to_string);
encode(
&header,
claims,
&EncodingKey::from_ed_pem(PRIVATE_KEY).unwrap(),
)
.unwrap()
}
fn verifier() -> GatewayJwtAuthenticator {
GatewayJwtAuthenticator::from_jwks(
JWKS,
"openshell-gateway:test",
"urn:openshell:extension:middleware:test",
)
.unwrap()
}
#[test]
fn authenticates_gateway_and_supervisor_callers() {
for kind in [
ExtensionCallerKind::Gateway,
ExtensionCallerKind::Supervisor,
] {
let expected = claims(kind);
let caller = verifier()
.authenticate(&token(&expected, Some(EXTENSION_JWT_TYP)))
.unwrap();
assert_eq!(caller.kind, kind);
assert_eq!(caller.subject, expected.sub);
assert_eq!(caller.sandbox_id, expected.sandbox_id);
}
}
#[test]
fn pem_and_jwks_constructors_accept_the_same_key() {
let verifier = GatewayJwtAuthenticator::from_ed25519_pem(
PUBLIC_KEY,
"test-key",
"openshell-gateway:test",
"urn:openshell:extension:middleware:test",
)
.unwrap();
assert!(
verifier
.authenticate(&token(
&claims(ExtensionCallerKind::Gateway),
Some(EXTENSION_JWT_TYP)
))
.is_ok()
);
}
#[test]
fn rejects_untyped_and_inconsistent_credentials() {
let verifier = verifier();
assert!(matches!(
verifier.authenticate(&token(&claims(ExtensionCallerKind::Gateway), None)),
Err(VerificationError::UnexpectedTokenType)
));
let mut inconsistent = claims(ExtensionCallerKind::Supervisor);
inconsistent.sub = "spiffe://openshell/sandbox/someone-else".into();
assert!(matches!(
verifier.authenticate(&token(&inconsistent, Some(EXTENSION_JWT_TYP))),
Err(VerificationError::InvalidClaims(_))
));
}
#[test]
fn request_helper_returns_stable_unauthenticated_errors() {
let verifier = verifier();
assert_eq!(
verifier
.authenticate_request(&Request::new(()))
.unwrap_err()
.code(),
tonic::Code::Unauthenticated
);
let mut request = Request::new(());
request
.metadata_mut()
.insert("authorization", "Bearer invalid".parse().unwrap());
let error = verifier.authenticate_request(&request).unwrap_err();
assert_eq!(error.code(), tonic::Code::Unauthenticated);
assert_eq!(error.message(), "invalid extension token");
}
}
+2
View File
@@ -34,6 +34,8 @@ pub mod client;
pub mod config;
pub mod edge_tunnel;
pub mod error;
#[cfg(feature = "extension")]
pub mod extension;
pub mod oidc;
pub mod raw;
pub mod refresh;
@@ -90,6 +90,8 @@ Return your expected audience in the `expected_audience` field of your `Describe
Provision the trusted gateway URL, expected gateway ID, and public key or JWKS through the deployment. This operator-provisioned key material is the authoritative cold-start trust anchor. The expected issuer is exactly `openshell-gateway:<gateway_id>`; fetching JWKS does not establish that identity by itself. After initial trust is established, `GET /.well-known/openid-configuration` and its `jwks_uri` provide steady-state key refresh and operational convenience. The document is OIDC-shaped rather than OIDC-compliant: `issuer` is the gateway identity, not the URL serving the document, so compare `iss` against the configured value and fetch updates only over authenticated TLS at the trusted gateway URL.
Rust services can enable the `extension` Cargo feature on `openshell-sdk` and use `openshell_sdk::extension::GatewayJwtAuthenticator`. Construct it from a trusted Ed25519 public key and `kid`, or from a trusted JWKS document, then call `authenticate_request` on each tonic request. The returned `AuthenticatedCaller` exposes the normalized caller kind, subject, and optional sandbox ID for RPC-specific authorization. Key refresh remains deployment-owned; rebuild the authenticator after accepting a refreshed key set from the trusted gateway URL.
Cache keys by `kid`. Validate, at minimum:
- `typ` is exactly `openshell-ext+jwt`. Extension tokens and sandbox-to-gateway bootstrap tokens share a signing key and differ only in audience; this header is a second, independent discriminator.
+5
View File
@@ -156,6 +156,11 @@ name = "proxy_egress_pipeline"
path = "tests/proxy_egress_pipeline.rs"
required-features = ["e2e-host-gateway"]
[[test]]
name = "supervisor_middleware"
path = "tests/supervisor_middleware.rs"
required-features = ["e2e-docker"]
[[test]]
name = "workspace_namespace_managed"
path = "tests/workspace_namespace_managed.rs"
+123
View File
@@ -0,0 +1,123 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Black-box happy-path coverage for authenticated supervisor middleware.
#![cfg(feature = "e2e-docker")]
use std::io::Write;
use openshell_e2e::harness::container::ContainerHttpServer;
use openshell_e2e::harness::sandbox::SandboxGuard;
use serde_json::Value;
use tempfile::NamedTempFile;
async fn start_test_server() -> Result<ContainerHttpServer, String> {
let script = r#"from http.server import BaseHTTPRequestHandler, HTTPServer
import json
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.end_headers()
def do_POST(self):
body = self.rfile.read(int(self.headers.get("Content-Length", "0")))
parsed = json.loads(body)
response = json.dumps({
"received_payload": parsed.get("payload"),
"fixture_header": self.headers.get("x-openshell-middleware-fixture"),
}, sort_keys=True).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(response)))
self.end_headers()
self.wfile.write(response)
def log_message(self, format, *args):
pass
HTTPServer(("0.0.0.0", 8000), Handler).serve_forever()
"#;
ContainerHttpServer::start_python("middleware-happy.openshell.test", script).await
}
fn write_policy(host: &str, port: u16) -> Result<NamedTempFile, String> {
let mut file = NamedTempFile::new().map_err(|error| format!("create policy: {error}"))?;
let policy = format!(
r#"version: 1
network_middlewares:
scripted-e2e:
name: Scripted E2E middleware
middleware: e2e-scripted
order: 10
config: {{}}
on_error: fail_closed
endpoints:
include:
- {host}
network_policies:
middleware_target:
name: middleware_target
endpoints:
- host: {host}
port: {port}
protocol: rest
enforcement: enforce
allowed_ips:
- "10.0.0.0/8"
- "172.0.0.0/8"
- "192.168.0.0/16"
- "fc00::/7"
rules:
- allow:
method: POST
path: "/inspect"
binaries:
- path: /usr/bin/python*
- path: /usr/local/bin/python*
- path: /sandbox/.uv/python/*/bin/python*
"#
);
file.write_all(policy.as_bytes())
.map_err(|error| format!("write policy: {error}"))?;
file.flush()
.map_err(|error| format!("flush policy: {error}"))?;
Ok(file)
}
#[tokio::test]
async fn authenticated_middleware_mutates_request_body_and_headers() {
let server = start_test_server().await.expect("start upstream server");
let policy = write_policy(&server.host, server.port).expect("write middleware policy");
let script = format!(
r#"import json, urllib.request
request = urllib.request.Request(
"http://{host}:{port}/inspect",
data=json.dumps({{"payload": "raw-secret"}}).encode(),
headers={{"Content-Type": "application/json"}},
method="POST",
)
with urllib.request.urlopen(request, timeout=15) as response:
print("MIDDLEWARE_RESULT=" + response.read().decode())
"#,
host = server.host,
port = server.port,
);
let policy_path = policy.path().to_string_lossy().into_owned();
let sandbox = SandboxGuard::create(&["--policy", &policy_path, "--", "python3", "-c", &script])
.await
.expect("create middleware sandbox");
let result = sandbox
.create_output
.lines()
.find_map(|line| line.split_once("MIDDLEWARE_RESULT=").map(|(_, json)| json))
.expect("sandbox output should contain the upstream response");
let result: Value = serde_json::from_str(result).expect("upstream response should be JSON");
assert_eq!(result["received_payload"], "[REDACTED]");
assert_eq!(result["fixture_header"], "evaluated");
}
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,31 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
# Standalone test fixture. Keeping it out of the root workspace prevents the
# fixture binary from becoming part of production workspace builds or images.
[workspace]
[package]
name = "openshell-e2e-middleware-fixture"
description = "Authenticated supervisor middleware used only by OpenShell E2E tests"
version = "0.1.0"
edition = "2024"
rust-version = "1.90"
license = "Apache-2.0"
publish = false
[dependencies]
openshell-core = { path = "../../crates/openshell-core", default-features = false }
openshell-sdk = { path = "../../crates/openshell-sdk", features = ["extension"] }
tokio = { version = "1.43", features = ["fs", "macros", "rt-multi-thread"] }
tonic = { version = "0.14", features = ["transport", "tls-ring"] }
[lints.rust]
unsafe_code = "warn"
rust_2018_idioms = { level = "warn", priority = -1 }
[lints.clippy]
all = { level = "warn", priority = -1 }
pedantic = { level = "warn", priority = -1 }
module_name_repetitions = "allow"
missing_errors_doc = "allow"
@@ -0,0 +1,188 @@
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0
//! Authenticated gRPC middleware used by the focused middleware E2E lane.
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use openshell_core::middleware::WebSocketResponseStream;
use openshell_core::proto::middleware::v1::supervisor_middleware_server::{
SupervisorMiddleware, SupervisorMiddlewareServer,
};
use openshell_core::proto::{
Decision, ExistingHeaderAction, HeaderMutation, HttpRequestEvaluation, HttpRequestResult,
MiddlewareBinding, MiddlewareManifest, SupervisorMiddlewareOperation,
SupervisorMiddlewarePhase, ValidateConfigRequest, ValidateConfigResponse,
WebSocketSessionEvent, WriteHeader, header_mutation,
};
use openshell_sdk::extension::{ExtensionCallerKind, GatewayJwtAuthenticator};
use tonic::transport::{Identity, Server, ServerTlsConfig};
use tonic::{Request, Response, Status};
const REGISTRATION_NAME: &str = "e2e-scripted";
const PHASE: SupervisorMiddlewarePhase = SupervisorMiddlewarePhase::PreCredentials;
const MAX_PAYLOAD_BYTES: u64 = 4 * 1024;
#[derive(Debug)]
struct Config {
listen: SocketAddr,
tls_cert: PathBuf,
tls_key: PathBuf,
gateway_public_key: PathBuf,
gateway_key_id: String,
gateway_issuer: String,
audience: String,
}
impl Config {
fn from_env() -> Result<Self, Box<dyn std::error::Error>> {
Ok(Self {
listen: required_env("OPENSHELL_E2E_MIDDLEWARE_LISTEN")?.parse()?,
tls_cert: required_env("OPENSHELL_E2E_MIDDLEWARE_TLS_CERT")?.into(),
tls_key: required_env("OPENSHELL_E2E_MIDDLEWARE_TLS_KEY")?.into(),
gateway_public_key: required_env("OPENSHELL_E2E_GATEWAY_PUBLIC_KEY")?.into(),
gateway_key_id: required_env("OPENSHELL_E2E_GATEWAY_KEY_ID")?,
gateway_issuer: required_env("OPENSHELL_E2E_GATEWAY_ISSUER")?,
audience: required_env("OPENSHELL_E2E_MIDDLEWARE_AUDIENCE")?,
})
}
}
fn required_env(name: &str) -> Result<String, std::env::VarError> {
std::env::var(name)
}
#[derive(Debug)]
struct ScriptedMiddleware {
authenticator: Arc<GatewayJwtAuthenticator>,
audience: String,
}
impl ScriptedMiddleware {
fn authenticate<T>(
&self,
request: &Request<T>,
allowed: &[ExtensionCallerKind],
) -> Result<(), Status> {
let caller = self.authenticator.authenticate_request(request)?;
if allowed.contains(&caller.kind) {
Ok(())
} else {
Err(Status::permission_denied(
"caller kind is not authorized for this middleware RPC",
))
}
}
}
#[tonic::async_trait]
impl SupervisorMiddleware for ScriptedMiddleware {
type EvaluateWebSocketSessionStream = WebSocketResponseStream;
async fn describe(&self, request: Request<()>) -> Result<Response<MiddlewareManifest>, Status> {
self.authenticate(
&request,
&[
ExtensionCallerKind::Gateway,
ExtensionCallerKind::Supervisor,
],
)?;
Ok(Response::new(MiddlewareManifest {
name: "openshell-e2e-middleware-fixture".into(),
service_version: env!("CARGO_PKG_VERSION").into(),
bindings: vec![MiddlewareBinding {
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
phase: PHASE as i32,
max_payload_bytes: MAX_PAYLOAD_BYTES,
timeout: String::new(),
}],
expected_audience: self.audience.clone(),
}))
}
async fn validate_config(
&self,
request: Request<ValidateConfigRequest>,
) -> Result<Response<ValidateConfigResponse>, Status> {
self.authenticate(&request, &[ExtensionCallerKind::Gateway])?;
let valid = request.get_ref().middleware_name == REGISTRATION_NAME;
Ok(Response::new(ValidateConfigResponse {
valid,
reason: if valid {
String::new()
} else {
"unsupported middleware registration".into()
},
}))
}
async fn evaluate_http_request(
&self,
request: Request<HttpRequestEvaluation>,
) -> Result<Response<HttpRequestResult>, Status> {
self.authenticate(&request, &[ExtensionCallerKind::Supervisor])?;
let request = request.into_inner();
if request.middleware_name != REGISTRATION_NAME || request.phase != PHASE as i32 {
return Err(Status::invalid_argument(
"unsupported middleware registration or phase",
));
}
let body = String::from_utf8(request.body)
.map_err(|_| Status::invalid_argument("fixture requires a UTF-8 request body"))?;
let replacement = body.replace("raw-secret", "[REDACTED]");
Ok(Response::new(HttpRequestResult {
decision: Decision::Allow as i32,
body: replacement.into_bytes(),
has_body: true,
header_mutations: vec![HeaderMutation {
operation: Some(header_mutation::Operation::Write(WriteHeader {
name: "x-openshell-middleware-fixture".into(),
value: "evaluated".into(),
on_existing: ExistingHeaderAction::Overwrite as i32,
})),
}],
..Default::default()
}))
}
async fn evaluate_web_socket_session(
&self,
_request: Request<tonic::Streaming<WebSocketSessionEvent>>,
) -> Result<Response<Self::EvaluateWebSocketSessionStream>, Status> {
Err(Status::unimplemented(
"the HTTP-only E2E fixture does not implement WebSocket middleware",
))
}
}
async fn read(path: &Path) -> Result<Vec<u8>, std::io::Error> {
tokio::fs::read(path).await
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
let config = Config::from_env()?;
let cert = read(&config.tls_cert).await?;
let key = read(&config.tls_key).await?;
let public_key = read(&config.gateway_public_key).await?;
let authenticator = GatewayJwtAuthenticator::from_ed25519_pem(
&public_key,
config.gateway_key_id,
config.gateway_issuer,
config.audience.clone(),
)?;
let service = ScriptedMiddleware {
authenticator: Arc::new(authenticator),
audience: config.audience,
};
Server::builder()
.tls_config(ServerTlsConfig::new().identity(Identity::from_pem(cert, key)))?
.add_service(SupervisorMiddlewareServer::new(service))
.serve(config.listen)
.await?;
Ok(())
}
+114
View File
@@ -118,6 +118,11 @@ E2E_NAMESPACE=""
DOCKER_NETWORK_NAME=""
DOCKER_NETWORK_CONNECTED_CONTAINER=""
DOCKER_NETWORK_MANAGED=0
MIDDLEWARE_FIXTURE_PID=""
MIDDLEWARE_FIXTURE_LOG="${WORKDIR}/middleware-fixture.log"
MIDDLEWARE_FIXTURE_ENDPOINT=""
MIDDLEWARE_FIXTURE_CA_CERT=""
MIDDLEWARE_FIXTURE_AUDIENCE="urn:openshell:extension:middleware:e2e-scripted"
GPU_MODE="${OPENSHELL_E2E_DOCKER_GPU:-0}"
OIDC_MODE="${OPENSHELL_E2E_OIDC_GATEWAY:-0}"
OIDC_ISSUER="${OPENSHELL_E2E_OIDC_ISSUER:-}"
@@ -141,6 +146,18 @@ cleanup() {
e2e_stop_gateway "${GATEWAY_PID}" "${GATEWAY_PID_FILE}"
e2e_stop_process "${DRIVER_PID}" "external Docker compute driver"
if [ -n "${MIDDLEWARE_FIXTURE_PID}" ] \
&& kill -0 "${MIDDLEWARE_FIXTURE_PID}" 2>/dev/null; then
kill "${MIDDLEWARE_FIXTURE_PID}" 2>/dev/null || true
wait "${MIDDLEWARE_FIXTURE_PID}" 2>/dev/null || true
fi
if [ "${exit_code}" -ne 0 ] && [ -s "${MIDDLEWARE_FIXTURE_LOG}" ]; then
echo "=== supervisor middleware fixture log ==="
cat "${MIDDLEWARE_FIXTURE_LOG}"
echo "=== end supervisor middleware fixture log ==="
fi
if [ "${exit_code}" -ne 0 ] \
&& [ -n "${E2E_NAMESPACE}" ] \
&& command -v docker >/dev/null 2>&1; then
@@ -262,6 +279,91 @@ connect_current_container_to_docker_network() {
GATEWAY_HOST_ALIAS_IP="${container_ip}"
}
start_middleware_fixture() {
local manifest="${ROOT}/e2e/supervisor-middleware-fixture/Cargo.toml"
local binary="${TARGET_DIR}/debug/openshell-e2e-middleware-fixture"
local fixture_dir="${STATE_DIR}/middleware-fixture"
local fixture_host
local fixture_port
echo "Building authenticated supervisor middleware E2E fixture..."
CARGO_TARGET_DIR="${TARGET_DIR}" cargo build --locked --manifest-path "${manifest}"
if [ -n "${GATEWAY_HOST_ALIAS_IP}" ]; then
fixture_host="${GATEWAY_HOST_ALIAS_IP}"
elif [ "$(uname -s)" = "Linux" ]; then
fixture_host="$(docker network inspect \
--format '{{(index .IPAM.Config 0).Gateway}}' \
"${DOCKER_NETWORK_NAME}")"
else
echo "ERROR: supervisor middleware E2E currently supports GitHub Actions and native Linux Docker." >&2
return 2
fi
if [ -z "${fixture_host}" ]; then
echo "ERROR: failed to determine a supervisor middleware fixture address." >&2
return 2
fi
mkdir -p "${fixture_dir}"
openssl req -x509 -newkey rsa:2048 -nodes \
-keyout "${fixture_dir}/ca.key" \
-out "${fixture_dir}/ca.crt" \
-days 1 \
-subj "/CN=OpenShell middleware E2E CA" \
-addext "basicConstraints=critical,CA:TRUE" \
-addext "keyUsage=critical,keyCertSign,cRLSign" >/dev/null 2>&1
openssl req -newkey rsa:2048 -nodes \
-keyout "${fixture_dir}/tls.key" \
-out "${fixture_dir}/tls.csr" \
-subj "/CN=${fixture_host}" >/dev/null 2>&1
{
printf 'basicConstraints=critical,CA:FALSE\n'
printf 'keyUsage=critical,digitalSignature,keyEncipherment\n'
printf 'extendedKeyUsage=serverAuth\n'
printf 'subjectAltName=IP:%s\n' "${fixture_host}"
} >"${fixture_dir}/tls.ext"
openssl x509 -req \
-in "${fixture_dir}/tls.csr" \
-CA "${fixture_dir}/ca.crt" \
-CAkey "${fixture_dir}/ca.key" \
-CAcreateserial \
-out "${fixture_dir}/tls.crt" \
-days 1 \
-extfile "${fixture_dir}/tls.ext" >/dev/null 2>&1
MIDDLEWARE_FIXTURE_CA_CERT="${fixture_dir}/ca.crt"
fixture_port="$(e2e_pick_port)"
echo "Starting supervisor middleware E2E fixture on ${fixture_host}:${fixture_port}..."
OPENSHELL_E2E_MIDDLEWARE_LISTEN="0.0.0.0:${fixture_port}" \
OPENSHELL_E2E_MIDDLEWARE_TLS_CERT="${fixture_dir}/tls.crt" \
OPENSHELL_E2E_MIDDLEWARE_TLS_KEY="${fixture_dir}/tls.key" \
OPENSHELL_E2E_GATEWAY_PUBLIC_KEY="${JWT_DIR}/public.pem" \
OPENSHELL_E2E_GATEWAY_KEY_ID="$(tr -d '\r\n' <"${JWT_DIR}/kid")" \
OPENSHELL_E2E_GATEWAY_ISSUER="openshell-gateway:openshell-e2e-docker-${HOST_PORT}" \
OPENSHELL_E2E_MIDDLEWARE_AUDIENCE="${MIDDLEWARE_FIXTURE_AUDIENCE}" \
"${binary}" >"${MIDDLEWARE_FIXTURE_LOG}" 2>&1 &
MIDDLEWARE_FIXTURE_PID=$!
local attempt
for attempt in $(seq 1 100); do
if ! kill -0 "${MIDDLEWARE_FIXTURE_PID}" 2>/dev/null; then
echo "ERROR: supervisor middleware E2E fixture exited before becoming ready." >&2
cat "${MIDDLEWARE_FIXTURE_LOG}" >&2 || true
return 1
fi
if python3 -c "import socket; s=socket.create_connection(('127.0.0.1', ${fixture_port}), 0.2); s.close()" 2>/dev/null; then
MIDDLEWARE_FIXTURE_ENDPOINT="https://${fixture_host}:${fixture_port}"
echo "Supervisor middleware E2E fixture ready at ${MIDDLEWARE_FIXTURE_ENDPOINT}."
return 0
fi
sleep 0.2
done
echo "ERROR: supervisor middleware E2E fixture did not become ready within 20s." >&2
return 1
}
if [ -n "${OPENSHELL_GATEWAY_ENDPOINT:-}" ]; then
case "${OPENSHELL_GATEWAY_ENDPOINT}" in
http://*) ;;
@@ -484,6 +586,9 @@ fi
echo "Starting openshell-gateway on port ${HOST_PORT} (namespace: ${E2E_NAMESPACE})..."
echo "Using sandbox image: ${SANDBOX_IMAGE} (pull policy: ${SANDBOX_IMAGE_PULL_POLICY})"
e2e_generate_gateway_jwt "${JWT_DIR}"
if [ "${OPENSHELL_E2E_MIDDLEWARE_FIXTURE:-0}" = "1" ]; then
start_middleware_fixture
fi
# Driver-specific options moved from CLI flags into a TOML config table
# (commit 560550d2). Synthesize a minimal config here and pass --config.
@@ -509,6 +614,15 @@ GATEWAY_CONFIG="${STATE_DIR}/gateway.toml"
e2e_write_gateway_oidc_config "${OPENSHELL_OIDC_ISSUER}"
fi
fi
if [ -n "${MIDDLEWARE_FIXTURE_ENDPOINT}" ]; then
printf '[[openshell.supervisor.middleware]]\n'
printf 'name = "e2e-scripted"\n'
printf 'grpc_endpoint = %s\n' "$(toml_string "${MIDDLEWARE_FIXTURE_ENDPOINT}")"
printf 'tls_ca_cert_path = %s\n' "$(toml_string "${MIDDLEWARE_FIXTURE_CA_CERT}")"
printf 'audience = %s\n' "$(toml_string "${MIDDLEWARE_FIXTURE_AUDIENCE}")"
printf 'max_payload_bytes = 4096\n'
printf 'timeout = "500ms"\n\n'
fi
printf '[openshell.drivers.docker]\n'
if [ "${OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER:-0}" = "1" ]; then
printf 'socket_path = %s\n' "$(toml_string "${DRIVER_SOCKET}")"
+6
View File
@@ -92,6 +92,12 @@ run = [
"e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test websocket_conformance",
]
["e2e:supervisor-middleware"]
description = "Run the authenticated supervisor middleware happy-path E2E test against a Docker-backed gateway"
run = [
"OPENSHELL_E2E_MIDDLEWARE_FIXTURE=1 e2e/with-docker-gateway.sh cargo test --manifest-path e2e/rust/Cargo.toml --features e2e-docker --test supervisor_middleware -- --nocapture",
]
["e2e:mcp"]
description = "Run MCP conformance e2e scenarios against one Docker-backed gateway (static defaults for spec 2025-11-25; set OPENSHELL_MCP_CONFORMANCE_SCENARIOS for a focused subset)"
run = "bash e2e/mcp-conformance.sh"