mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-25 06:42:52 +08:00
krishicks
4b7486e73d
Tests for the gateway's OCSF events each carried their own copy of the same fixtures: converting captured events to JSON, the request ID and peer address, running a handler inside a routed request, and checking event shapes. They now share them: - OcsfCapture::json_events returns captured events as their exported JSON. - auth::audit::test_support holds the test request origin, audited() for running a handler as a routed request, and assertions for authentication failures, logons, denials, boundary findings, the request origin, and redacted secrets. The event assertions also check each event against its OCSF class schema, which the handler tests did not do before. - auth::principal::test_support builds the OIDC user and gateway-JWT sandbox principals that several test modules defined identically. Refs #2912 Signed-off-by: Kris Hicks <khicks@nvidia.com>
4797 lines
184 KiB
Rust
4797 lines
184 KiB
Rust
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
//! Protocol multiplexing for gRPC and HTTP on the same port.
|
|
//!
|
|
//! This module implements connection-level multiplexing that routes requests
|
|
//! to either the gRPC service or HTTP endpoints based on the request headers.
|
|
|
|
use bytes::{Bytes, BytesMut};
|
|
use http::{Extensions, HeaderValue, Request, Response};
|
|
use http_body::Body;
|
|
use http_body_util::{BodyExt, Full, LengthLimitError, Limited, StreamBody};
|
|
use hyper::body::Incoming;
|
|
use hyper_util::{
|
|
rt::{TokioExecutor, TokioIo, TokioTimer},
|
|
server::conn::auto::Builder,
|
|
service::TowerToHyperService,
|
|
};
|
|
use metrics::{counter, histogram};
|
|
use openshell_core::proto::open_shell_server::OpenShellServer;
|
|
use openshell_core::{
|
|
Config,
|
|
proto::{Provider, UpdateProviderRequest},
|
|
};
|
|
use openshell_gateway_interceptors::{EvaluationContext, GatewayInterceptorRuntime};
|
|
use openshell_otel::HeaderMapExtractor;
|
|
use opentelemetry::propagation::TextMapPropagator;
|
|
use opentelemetry::trace::TraceContextExt as _;
|
|
use opentelemetry_sdk::propagation::TraceContextPropagator;
|
|
use prost::Message;
|
|
use std::collections::BTreeMap;
|
|
use std::convert::Infallible;
|
|
use std::future::Future;
|
|
use std::pin::Pin;
|
|
use std::sync::{Arc, Mutex};
|
|
use std::task::{Context, Poll};
|
|
use std::time::{Duration, Instant};
|
|
use tokio::io::{AsyncRead, AsyncWrite};
|
|
use tower::ServiceExt;
|
|
use tower_http::request_id::{MakeRequestId, RequestId};
|
|
use tracing::{Span, warn};
|
|
use tracing_opentelemetry::OpenTelemetrySpanExt as _;
|
|
|
|
use crate::{
|
|
OpenShellService, ServerState,
|
|
auth::audit::{self, AuthzDenial, ConnectionSource, RequestOrigin},
|
|
auth::authenticator::{AuthFailureReason, AuthMechanism, Authentication, AuthenticatorChain},
|
|
auth::authz::AuthzPolicy,
|
|
auth::identity::Identity,
|
|
auth::oidc::{self, OidcAuthenticator},
|
|
auth::principal::{OperatorPrincipal, Principal, UserPrincipal},
|
|
auth::workspace_authz::{MinWorkspaceRole, authorize_workspace},
|
|
http_router, service_http_router,
|
|
};
|
|
|
|
/// Request-ID generator that produces a UUID v4 for each inbound request.
|
|
#[derive(Clone)]
|
|
struct UuidRequestId;
|
|
|
|
impl MakeRequestId for UuidRequestId {
|
|
fn make_request_id<B>(&mut self, _req: &Request<B>) -> Option<RequestId> {
|
|
let id = uuid::Uuid::new_v4().to_string();
|
|
Some(RequestId::new(HeaderValue::from_str(&id).unwrap()))
|
|
}
|
|
}
|
|
|
|
/// Paths called on a timer rather than by someone waiting on the result.
|
|
const POLLED_PATHS: &[&str] = &[
|
|
"/health",
|
|
"/healthz",
|
|
"/readyz",
|
|
"/openshell.v1.OpenShell/GetSandboxConfig",
|
|
"/openshell.v1.OpenShell/ReportProviderReadiness",
|
|
"/openshell.v1.OpenShell/PeerReportProviderReadiness",
|
|
];
|
|
|
|
/// Build a tracing span for an inbound request, recording the `request_id`
|
|
/// header (set by [`UuidRequestId`] or supplied by the client).
|
|
fn make_request_span<B>(req: &Request<B>) -> Span {
|
|
let path = req.uri().path();
|
|
let request_id = req
|
|
.headers()
|
|
.get("x-request-id")
|
|
.and_then(|v| v.to_str().ok())
|
|
.unwrap_or("-");
|
|
|
|
// `otel.name` and `otel.kind` are consumed by `tracing-opentelemetry` to
|
|
// set the exported span's name and kind; they are not emitted as
|
|
// attributes. See [`otel_span_name`] for why the name cannot simply be
|
|
// the callsite name.
|
|
let otel_name = otel_span_name(req.method(), path);
|
|
|
|
let span = if POLLED_PATHS.contains(&path) {
|
|
tracing::debug_span!(
|
|
"request",
|
|
method = %req.method(),
|
|
path,
|
|
request_id,
|
|
otel.name = %otel_name,
|
|
otel.kind = "server",
|
|
otel.status_code = tracing::field::Empty,
|
|
http.response.status_code = tracing::field::Empty,
|
|
)
|
|
} else {
|
|
let span = tracing::info_span!(
|
|
"request",
|
|
method = %req.method(),
|
|
path,
|
|
request_id,
|
|
otel.name = %otel_name,
|
|
otel.kind = "server",
|
|
otel.status_code = tracing::field::Empty,
|
|
http.response.status_code = tracing::field::Empty,
|
|
rpc.system = tracing::field::Empty,
|
|
rpc.service = tracing::field::Empty,
|
|
rpc.method = tracing::field::Empty,
|
|
rpc.grpc.status_code = tracing::field::Empty,
|
|
);
|
|
// RPC-aware backends build service maps from these; without them a
|
|
// gRPC call is just an HTTP span.
|
|
if let Some((service, method)) = grpc_service_method(path) {
|
|
span.record("rpc.system", "grpc");
|
|
span.record("rpc.service", service);
|
|
span.record("rpc.method", method);
|
|
}
|
|
span
|
|
};
|
|
|
|
let propagator = TraceContextPropagator::new();
|
|
let parent = propagator.extract_with_context(
|
|
&opentelemetry::Context::new(),
|
|
&HeaderMapExtractor::new(req.headers()),
|
|
);
|
|
if parent.span().span_context().is_valid() {
|
|
let _ = span.set_parent(parent);
|
|
}
|
|
|
|
span
|
|
}
|
|
|
|
/// Log response status and latency, record protocol status, and mark failures.
|
|
fn log_response<B>(res: &Response<B>, latency: Duration, span: &Span) {
|
|
let status = res.status();
|
|
span.record("http.response.status_code", status.as_u16());
|
|
record_grpc_status(res.headers(), span);
|
|
if status.is_server_error() {
|
|
crate::otel_tracing::mark_error(span);
|
|
}
|
|
// Polled requests get a DEBUG span, which is `None` when filtered out.
|
|
let polled = span
|
|
.metadata()
|
|
.is_none_or(|m| *m.level() == tracing::Level::DEBUG);
|
|
let server_error = status.is_server_error();
|
|
let (status, latency_ms) = (status.as_u16(), latency.as_millis());
|
|
match (polled, server_error) {
|
|
(false, _) => tracing::info!(status, latency_ms, "response"),
|
|
(true, true) => tracing::warn!(status, latency_ms, "response"),
|
|
(true, false) => tracing::debug!(status, latency_ms, "response"),
|
|
}
|
|
}
|
|
|
|
fn record_response_trailers(
|
|
trailers: Option<&http::HeaderMap>,
|
|
_stream_duration: Duration,
|
|
span: &Span,
|
|
) {
|
|
if let Some(trailers) = trailers {
|
|
record_grpc_status(trailers, span);
|
|
}
|
|
}
|
|
|
|
fn record_grpc_status(headers: &http::HeaderMap, span: &Span) {
|
|
let Some(code) = headers
|
|
.get("grpc-status")
|
|
.and_then(|value| value.to_str().ok())
|
|
.and_then(|value| value.parse::<i64>().ok())
|
|
else {
|
|
return;
|
|
};
|
|
|
|
span.record("rpc.grpc.status_code", code);
|
|
if code != 0 {
|
|
crate::otel_tracing::mark_error(span);
|
|
}
|
|
}
|
|
|
|
/// Wrap a service with the standard request-ID middleware stack.
|
|
///
|
|
/// Layer order: `SetRequestId` → `TraceLayer` → `PropagateRequestId`.
|
|
macro_rules! request_id_middleware {
|
|
($service:expr) => {{
|
|
let x_request_id = ::http::HeaderName::from_static("x-request-id");
|
|
::tower::ServiceBuilder::new()
|
|
.layer(::tower_http::request_id::SetRequestIdLayer::new(
|
|
x_request_id.clone(),
|
|
UuidRequestId,
|
|
))
|
|
.layer(
|
|
::tower_http::trace::TraceLayer::new_for_http()
|
|
.make_span_with(make_request_span)
|
|
.on_request(())
|
|
.on_response(log_response)
|
|
.on_eos(record_response_trailers),
|
|
)
|
|
.layer(::tower_http::request_id::PropagateRequestIdLayer::new(
|
|
x_request_id,
|
|
))
|
|
.service($service)
|
|
}};
|
|
}
|
|
|
|
/// Maximum inbound gRPC message size (1 MB).
|
|
///
|
|
/// Replaces tonic's implicit 4 MB default with a conservative limit to
|
|
/// bound memory allocation from a single request. Sandbox creation is
|
|
/// the largest payload and well within this cap under normal use.
|
|
const MAX_GRPC_DECODE_SIZE: usize = 1_048_576;
|
|
const MAX_INTERCEPTED_GRPC_BODY_SIZE: usize = MAX_GRPC_DECODE_SIZE + 5;
|
|
/// Concurrent HTTP/2 streams allowed per connection. Sits above the
|
|
/// per-replica pending relay budget so pooled peer connections are bounded by
|
|
/// the relay caps rather than by the transport.
|
|
const MAX_HTTP2_CONCURRENT_STREAMS: u32 = 1024;
|
|
|
|
/// Multiplexed gRPC/HTTP service.
|
|
#[derive(Clone)]
|
|
pub struct MultiplexService {
|
|
state: Arc<ServerState>,
|
|
}
|
|
|
|
impl MultiplexService {
|
|
/// Create a new multiplex service.
|
|
#[must_use]
|
|
#[allow(clippy::missing_const_for_fn)]
|
|
pub fn new(state: Arc<ServerState>) -> Self {
|
|
Self { state }
|
|
}
|
|
|
|
/// Serve a connection, routing to gRPC or HTTP based on content-type.
|
|
pub async fn serve<S>(
|
|
&self,
|
|
stream: S,
|
|
source: ConnectionSource,
|
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
|
where
|
|
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
|
{
|
|
self.serve_with_peer_identity(stream, None, source).await
|
|
}
|
|
|
|
/// Serve a TLS connection with an optional mTLS peer identity.
|
|
pub async fn serve_with_peer_identity<S>(
|
|
&self,
|
|
stream: S,
|
|
peer_identity: Option<Identity>,
|
|
source: ConnectionSource,
|
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
|
where
|
|
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
|
{
|
|
self.serve_with_verified_certificate(stream, peer_identity, None, source)
|
|
.await
|
|
}
|
|
|
|
/// TLS listener only: fingerprint and identity must originate from its verified handshake.
|
|
pub async fn serve_with_verified_certificate<S>(
|
|
&self,
|
|
stream: S,
|
|
peer_identity: Option<Identity>,
|
|
certificate_sha256: Option<String>,
|
|
source: ConnectionSource,
|
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
|
where
|
|
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
|
{
|
|
let operator = if self.state.config.mtls_auth.operator_enabled {
|
|
peer_identity
|
|
.as_ref()
|
|
.filter(|identity| {
|
|
identity.provider == crate::auth::identity::IdentityProvider::Mtls
|
|
&& identity.roles.iter().any(|role| role == "operator")
|
|
})
|
|
.zip(certificate_sha256.clone())
|
|
.map(|(identity, certificate_sha256)| OperatorPrincipal {
|
|
identity: identity.clone(),
|
|
certificate_sha256,
|
|
})
|
|
} else {
|
|
None
|
|
};
|
|
let openshell = OpenShellServer::new(OpenShellService::new(self.state.clone()))
|
|
.max_decoding_message_size(MAX_GRPC_DECODE_SIZE);
|
|
let openshell = GatewayInterceptorGrpcService::new(
|
|
openshell,
|
|
self.state.gateway_interceptors.clone(),
|
|
Some(self.state.clone()),
|
|
);
|
|
let reflection = self.state.reflection_service.clone();
|
|
let authz_policy = self.state.config.oidc.as_ref().map(|oidc| AuthzPolicy {
|
|
admin_role: oidc.admin_role.clone(),
|
|
user_role: oidc.user_role.clone(),
|
|
scopes_enabled: !oidc.scopes_claim.is_empty(),
|
|
});
|
|
let authenticator_chain = build_authenticator_chain(&self.state);
|
|
let grpc_service = AuthGrpcRouter::with_peer_identity(
|
|
GrpcRouter::new(openshell, reflection),
|
|
authenticator_chain,
|
|
authz_policy,
|
|
self.state
|
|
.config
|
|
.mtls_auth
|
|
.enabled
|
|
.then_some(peer_identity)
|
|
.flatten(),
|
|
self.state.config.mtls_auth.enabled,
|
|
self.state.config.auth.allow_unauthenticated_users,
|
|
)
|
|
.with_peer_certificate(certificate_sha256)
|
|
.with_operator(operator)
|
|
.with_connection_source(source)
|
|
.with_logons(self.state.logons.clone());
|
|
let grpc_service =
|
|
GrpcRateLimitService::new(grpc_service, self.state.grpc_rate_limiter.clone());
|
|
let http_service = http_router(self.state.clone());
|
|
|
|
let grpc_service = request_id_middleware!(grpc_service);
|
|
let http_service = request_id_middleware!(http_service);
|
|
|
|
let service = MultiplexedService::new(grpc_service, http_service);
|
|
|
|
let mut builder = Builder::new(TokioExecutor::new());
|
|
// Server-side HTTP/2 keepalive: supervisors hold long-lived sessions, and without
|
|
// it the gateway never PINGs them, so idle/half-dead connections linger and orphan
|
|
// in-flight relay execs. The timer is required — hyper panics on the keepalive
|
|
// interval without one.
|
|
//
|
|
// Peer relays from one replica now share a single pooled connection, so every
|
|
// forwarded session for every sandbox counts against this one limit. hyper's
|
|
// default of 200 would cap the whole replica pair below MAX_PENDING_RELAYS.
|
|
builder
|
|
.http2()
|
|
.timer(TokioTimer::new())
|
|
.adaptive_window(true)
|
|
.max_concurrent_streams(MAX_HTTP2_CONCURRENT_STREAMS)
|
|
.keep_alive_interval(Some(Duration::from_secs(20)))
|
|
.keep_alive_timeout(Duration::from_secs(10));
|
|
|
|
builder
|
|
.serve_connection_with_upgrades(TokioIo::new(stream), service)
|
|
.await?;
|
|
|
|
Ok(())
|
|
}
|
|
|
|
/// Serve a plaintext HTTP connection for sandbox service endpoints only.
|
|
pub async fn serve_service_http<S>(
|
|
&self,
|
|
stream: S,
|
|
) -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
|
where
|
|
S: AsyncRead + AsyncWrite + Unpin + Send + 'static,
|
|
{
|
|
let http_service = TowerToHyperService::new(request_id_middleware!(service_http_router(
|
|
self.state.clone()
|
|
)));
|
|
|
|
Builder::new(TokioExecutor::new())
|
|
.serve_connection_with_upgrades(TokioIo::new(stream), http_service)
|
|
.await?;
|
|
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
/// `OpenShell` gRPC wrapper that applies configured gateway interceptors before
|
|
/// tonic dispatches to a specific RPC handler.
|
|
#[derive(Clone)]
|
|
struct GatewayInterceptorGrpcService<S> {
|
|
inner: S,
|
|
interceptors: Option<GatewayInterceptorRuntime>,
|
|
state: Option<Arc<ServerState>>,
|
|
}
|
|
|
|
impl<S> GatewayInterceptorGrpcService<S> {
|
|
fn new(
|
|
inner: S,
|
|
interceptors: Option<GatewayInterceptorRuntime>,
|
|
state: Option<Arc<ServerState>>,
|
|
) -> Self {
|
|
Self {
|
|
inner,
|
|
interceptors,
|
|
state,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<S> tower::Service<Request<BoxBody>> for GatewayInterceptorGrpcService<S>
|
|
where
|
|
S: tower::Service<Request<BoxBody>, Response = Response<tonic::body::Body>>
|
|
+ Clone
|
|
+ Send
|
|
+ 'static,
|
|
S::Future: Send + 'static,
|
|
S::Error: Send + 'static,
|
|
{
|
|
type Response = S::Response;
|
|
type Error = S::Error;
|
|
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
|
|
|
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
|
self.inner.poll_ready(cx)
|
|
}
|
|
|
|
fn call(&mut self, req: Request<BoxBody>) -> Self::Future {
|
|
let interceptors = self.interceptors.clone();
|
|
let state = self.state.clone();
|
|
let mut inner = self.inner.clone();
|
|
|
|
Box::pin(async move {
|
|
let Some(interceptors) = interceptors else {
|
|
return inner.ready().await?.call(req).await;
|
|
};
|
|
|
|
let path = req.uri().path().to_string();
|
|
if !interceptors.should_intercept_path(&path) {
|
|
return inner.ready().await?.call(req).await;
|
|
}
|
|
|
|
let context = gateway_interceptor_context(req.extensions());
|
|
let principal = req.extensions().get::<Principal>().cloned();
|
|
let (mut parts, body) = req.into_parts();
|
|
let mut body = match collect_intercepted_grpc_body(body).await {
|
|
Ok(body) => body,
|
|
Err(status) => return Ok(status.into_http()),
|
|
};
|
|
// Retain only in memory. evaluate_request below validates the single
|
|
// uncompressed frame before this extension reaches typed dispatch.
|
|
let original_body = body.clone();
|
|
if let Some(state) = state.as_ref() {
|
|
body =
|
|
match hydrate_update_provider_identity(&path, body, state, principal.as_ref())
|
|
.await
|
|
{
|
|
Ok(body) => body,
|
|
Err(status) => return Ok(status.into_http()),
|
|
};
|
|
}
|
|
|
|
let intercepted = match interceptors.evaluate_request(&path, &body, &context).await {
|
|
Ok(intercepted) => intercepted,
|
|
Err(status) => return Ok(status.into_http()),
|
|
};
|
|
|
|
parts
|
|
.extensions
|
|
.insert(crate::grpc::mutation_replay::OriginalMutation(
|
|
original_body[GRPC_FRAME_HEADER_LEN..].to_vec(),
|
|
));
|
|
|
|
let req = Request::from_parts(
|
|
parts,
|
|
boxed_body_from_bytes(Bytes::from(intercepted.body.clone())),
|
|
);
|
|
let response = inner.ready().await?.call(req).await?;
|
|
|
|
if grpc_status_from_response(&response) != "0"
|
|
|| response
|
|
.headers()
|
|
.get("openshell-replayed")
|
|
.is_some_and(|value| value == "true")
|
|
|| !interceptors.has_post_commit(&intercepted)
|
|
{
|
|
return Ok(response);
|
|
}
|
|
|
|
let (response, observation) = observe_intercepted_grpc_response(response).await;
|
|
let (response_body, trailers) = match observation {
|
|
Ok(observation) => observation,
|
|
Err(error) => {
|
|
warn!(
|
|
error = %error,
|
|
"gateway post-commit response observation failed; preserving committed response"
|
|
);
|
|
counter!(
|
|
"openshell_gateway_interceptor_post_commit_observation_failures_total",
|
|
"stage" => "response_body"
|
|
)
|
|
.increment(1);
|
|
return Ok(response);
|
|
}
|
|
};
|
|
if grpc_status_from_response_and_trailers(&response, trailers.as_ref()) == "0"
|
|
&& let Err(status) = interceptors
|
|
.evaluate_post_commit(&intercepted, &response_body, &context)
|
|
.await
|
|
{
|
|
warn!(
|
|
error = %status,
|
|
"gateway post-commit evaluation failed; preserving committed response"
|
|
);
|
|
counter!(
|
|
"openshell_gateway_interceptor_post_commit_observation_failures_total",
|
|
"stage" => "evaluation"
|
|
)
|
|
.increment(1);
|
|
}
|
|
|
|
Ok(response)
|
|
})
|
|
}
|
|
}
|
|
|
|
const UPDATE_PROVIDER_PATH: &str = "/openshell.v1.OpenShell/UpdateProvider";
|
|
const GRPC_FRAME_HEADER_LEN: usize = 5;
|
|
|
|
/// Complete immutable provider identity before policy interception.
|
|
///
|
|
/// Update requests intentionally omit immutable fields. Loading them here keeps
|
|
/// `provider update` a write-only operation while giving policy interceptors a
|
|
/// canonical proposed operation derived from trusted gateway state.
|
|
async fn hydrate_update_provider_identity(
|
|
path: &str,
|
|
body: Bytes,
|
|
state: &ServerState,
|
|
principal: Option<&Principal>,
|
|
) -> Result<Bytes, tonic::Status> {
|
|
if path != UPDATE_PROVIDER_PATH {
|
|
return Ok(body);
|
|
}
|
|
|
|
let mut request = decode_unary_grpc_message::<UpdateProviderRequest>(&body)?;
|
|
let Some(provider) = request.provider.as_mut() else {
|
|
return Ok(body);
|
|
};
|
|
if !provider.r#type.is_empty() && !provider.profile_workspace.is_empty() {
|
|
return Ok(body);
|
|
}
|
|
let name = provider
|
|
.metadata
|
|
.as_ref()
|
|
.map(|metadata| metadata.name.as_str())
|
|
.unwrap_or_default();
|
|
if name.is_empty() {
|
|
return Ok(body);
|
|
}
|
|
|
|
let principal =
|
|
principal.ok_or_else(|| tonic::Status::unauthenticated("authentication required"))?;
|
|
let authorized = authorize_workspace(
|
|
state.store.as_ref(),
|
|
&state.admin_role,
|
|
principal,
|
|
crate::auth::workspace_authz::selected_workspace_name(request.workspace_scope.as_ref())?,
|
|
MinWorkspaceRole::Admin,
|
|
)
|
|
.await?;
|
|
let workspace =
|
|
crate::grpc::workspace::resolve_workspace(state.store.as_ref(), &authorized.workspace)
|
|
.await?
|
|
.name;
|
|
let existing = state
|
|
.store
|
|
.get_message_by_name::<Provider>(&workspace, name)
|
|
.await
|
|
.map_err(|error| tonic::Status::internal(format!("provider lookup failed: {error}")))?
|
|
.ok_or_else(|| tonic::Status::not_found(format!("provider '{name}' not found")))?;
|
|
|
|
if provider.r#type.is_empty() {
|
|
provider.r#type = existing.r#type;
|
|
}
|
|
if provider.profile_workspace.is_empty() {
|
|
provider.profile_workspace = existing.profile_workspace;
|
|
}
|
|
|
|
encode_unary_grpc_message(&request)
|
|
}
|
|
|
|
fn decode_unary_grpc_message<M>(body: &[u8]) -> Result<M, tonic::Status>
|
|
where
|
|
M: Message + Default,
|
|
{
|
|
if body.len() < GRPC_FRAME_HEADER_LEN {
|
|
return Err(tonic::Status::invalid_argument("gRPC frame is too short"));
|
|
}
|
|
if body[0] != 0 {
|
|
return Err(tonic::Status::unimplemented(
|
|
"gateway interceptors do not support compressed gRPC frames",
|
|
));
|
|
}
|
|
let message_len = u32::from_be_bytes([body[1], body[2], body[3], body[4]]) as usize;
|
|
if body.len() != GRPC_FRAME_HEADER_LEN + message_len {
|
|
return Err(tonic::Status::invalid_argument(
|
|
"gRPC body must contain exactly one frame",
|
|
));
|
|
}
|
|
M::decode(&body[GRPC_FRAME_HEADER_LEN..])
|
|
.map_err(|error| tonic::Status::invalid_argument(format!("invalid gRPC message: {error}")))
|
|
}
|
|
|
|
fn encode_unary_grpc_message<M: Message>(message: &M) -> Result<Bytes, tonic::Status> {
|
|
let message = message.encode_to_vec();
|
|
let message_len = u32::try_from(message.len())
|
|
.map_err(|_| tonic::Status::resource_exhausted("gRPC message exceeds u32"))?;
|
|
let mut frame = Vec::with_capacity(GRPC_FRAME_HEADER_LEN + message.len());
|
|
frame.push(0);
|
|
frame.extend_from_slice(&message_len.to_be_bytes());
|
|
frame.extend_from_slice(&message);
|
|
Ok(Bytes::from(frame))
|
|
}
|
|
|
|
async fn collect_intercepted_grpc_body(body: BoxBody) -> Result<Bytes, tonic::Status> {
|
|
Limited::new(body, MAX_INTERCEPTED_GRPC_BODY_SIZE)
|
|
.collect()
|
|
.await
|
|
.map(http_body_util::Collected::to_bytes)
|
|
.map_err(|err| {
|
|
if err.downcast_ref::<LengthLimitError>().is_some() {
|
|
tonic::Status::resource_exhausted(format!(
|
|
"gRPC request body exceeds interceptor evaluation limit of {MAX_INTERCEPTED_GRPC_BODY_SIZE} bytes"
|
|
))
|
|
} else {
|
|
tonic::Status::internal(format!(
|
|
"failed to read gRPC request body for interceptor evaluation: {err}"
|
|
))
|
|
}
|
|
})
|
|
}
|
|
|
|
fn boxed_body_from_bytes(bytes: Bytes) -> BoxBody {
|
|
let body = Full::new(bytes)
|
|
.map_err(|never: Infallible| -> Box<dyn std::error::Error + Send + Sync> { match never {} })
|
|
.boxed_unsync();
|
|
BoxBody(body)
|
|
}
|
|
|
|
async fn observe_intercepted_grpc_response(
|
|
response: Response<tonic::body::Body>,
|
|
) -> (
|
|
Response<tonic::body::Body>,
|
|
Result<(Bytes, Option<http::HeaderMap>), String>,
|
|
) {
|
|
let (parts, mut body) = response.into_parts();
|
|
let mut frames = Vec::new();
|
|
let mut bytes = BytesMut::new();
|
|
let mut trailers = None;
|
|
|
|
while let Some(frame) = body.frame().await {
|
|
match frame {
|
|
Ok(frame) => {
|
|
if let Some(data) = frame.data_ref() {
|
|
bytes.extend_from_slice(data);
|
|
}
|
|
if let Some(frame_trailers) = frame.trailers_ref() {
|
|
trailers = Some(frame_trailers.clone());
|
|
}
|
|
frames.push(Ok(frame));
|
|
}
|
|
Err(status) => {
|
|
let error =
|
|
format!("failed to read gRPC response for post-commit evaluation: {status}");
|
|
frames.push(Err(status));
|
|
return (
|
|
Response::from_parts(parts, tonic_body_from_frames(frames)),
|
|
Err(error),
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
(
|
|
Response::from_parts(parts, tonic_body_from_frames(frames)),
|
|
Ok((bytes.freeze(), trailers)),
|
|
)
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn tonic_body_from_bytes_and_trailers(
|
|
bytes: Bytes,
|
|
trailers: Option<http::HeaderMap>,
|
|
) -> tonic::body::Body {
|
|
let mut frames: Vec<Result<http_body::Frame<Bytes>, tonic::Status>> = Vec::with_capacity(2);
|
|
if !bytes.is_empty() {
|
|
frames.push(Ok(http_body::Frame::data(bytes)));
|
|
}
|
|
if let Some(trailers) = trailers {
|
|
frames.push(Ok(http_body::Frame::trailers(trailers)));
|
|
}
|
|
tonic_body_from_frames(frames)
|
|
}
|
|
|
|
fn tonic_body_from_frames(
|
|
frames: Vec<Result<http_body::Frame<Bytes>, tonic::Status>>,
|
|
) -> tonic::body::Body {
|
|
tonic::body::Body::new(StreamBody::new(futures::stream::iter(frames)))
|
|
}
|
|
|
|
fn gateway_interceptor_context(extensions: &Extensions) -> EvaluationContext {
|
|
EvaluationContext {
|
|
principal: extensions
|
|
.get::<Principal>()
|
|
.map_or_else(unknown_gateway_principal, gateway_principal_fields),
|
|
validate_current_state: None,
|
|
}
|
|
}
|
|
|
|
fn gateway_principal_fields(principal: &Principal) -> BTreeMap<String, String> {
|
|
use crate::auth::principal::SandboxIdentitySource;
|
|
|
|
let mut fields = BTreeMap::new();
|
|
match principal {
|
|
Principal::User(user) => {
|
|
fields.insert("kind".to_string(), "user".to_string());
|
|
fields.insert("subject".to_string(), user.identity.subject.clone());
|
|
if let Some(display_name) = &user.identity.display_name {
|
|
fields.insert("display_name".to_string(), display_name.clone());
|
|
}
|
|
fields.insert(
|
|
"provider".to_string(),
|
|
identity_provider_name(user.identity.provider).to_string(),
|
|
);
|
|
if !user.identity.roles.is_empty() {
|
|
fields.insert("roles".to_string(), user.identity.roles.join(","));
|
|
}
|
|
if !user.identity.scopes.is_empty() {
|
|
fields.insert("scopes".to_string(), user.identity.scopes.join(","));
|
|
}
|
|
}
|
|
Principal::Operator(operator) => {
|
|
fields.insert("kind".to_string(), "operator".to_string());
|
|
fields.insert("subject".to_string(), operator.identity.subject.clone());
|
|
fields.insert("provider".to_string(), "mtls".to_string());
|
|
fields.insert(
|
|
"certificate_sha256".to_string(),
|
|
operator.certificate_sha256.clone(),
|
|
);
|
|
}
|
|
Principal::Sandbox(sandbox) => {
|
|
fields.insert("kind".to_string(), "sandbox".to_string());
|
|
fields.insert("sandbox_id".to_string(), sandbox.sandbox_id.clone());
|
|
fields.insert(
|
|
"source".to_string(),
|
|
match &sandbox.source {
|
|
SandboxIdentitySource::BootstrapJwt { .. } => "bootstrap_jwt",
|
|
SandboxIdentitySource::ComputeDriver { .. } => "compute_driver",
|
|
}
|
|
.to_string(),
|
|
);
|
|
if let Some(trust_domain) = &sandbox.trust_domain {
|
|
fields.insert("trust_domain".to_string(), trust_domain.clone());
|
|
}
|
|
}
|
|
Principal::Peer(peer) => {
|
|
fields.insert("kind".to_string(), "peer".to_string());
|
|
fields.insert("replica_id".to_string(), peer.replica_id.clone());
|
|
fields.insert("pod_uid".to_string(), peer.pod_uid.clone());
|
|
}
|
|
Principal::Anonymous => {
|
|
fields.insert("kind".to_string(), "anonymous".to_string());
|
|
}
|
|
}
|
|
fields
|
|
}
|
|
|
|
fn unknown_gateway_principal() -> BTreeMap<String, String> {
|
|
BTreeMap::from([("kind".to_string(), "unknown".to_string())])
|
|
}
|
|
|
|
fn identity_provider_name(provider: crate::auth::identity::IdentityProvider) -> &'static str {
|
|
match provider {
|
|
crate::auth::identity::IdentityProvider::Oidc => "oidc",
|
|
crate::auth::identity::IdentityProvider::Mtls => "mtls",
|
|
crate::auth::identity::IdentityProvider::CloudflareAccess => "cloudflare_access",
|
|
crate::auth::identity::IdentityProvider::LocalDev => "local_dev",
|
|
}
|
|
}
|
|
|
|
#[derive(Clone, Debug)]
|
|
pub struct GrpcRateLimiter {
|
|
requests: u64,
|
|
window: Duration,
|
|
state: Arc<Mutex<GrpcRateLimitState>>,
|
|
}
|
|
|
|
#[derive(Debug)]
|
|
struct GrpcRateLimitState {
|
|
window_started: Instant,
|
|
remaining: u64,
|
|
}
|
|
|
|
impl GrpcRateLimiter {
|
|
pub fn from_config(config: &Config) -> Option<Self> {
|
|
let (requests, window) = config.grpc_rate_limit()?;
|
|
Some(Self {
|
|
requests,
|
|
window,
|
|
state: Arc::new(Mutex::new(GrpcRateLimitState {
|
|
window_started: Instant::now(),
|
|
remaining: requests,
|
|
})),
|
|
})
|
|
}
|
|
|
|
pub(crate) fn allow(&self) -> bool {
|
|
let now = Instant::now();
|
|
let mut state = self
|
|
.state
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
if now.duration_since(state.window_started) >= self.window {
|
|
state.window_started = now;
|
|
state.remaining = self.requests;
|
|
}
|
|
if state.remaining == 0 {
|
|
false
|
|
} else {
|
|
state.remaining -= 1;
|
|
true
|
|
}
|
|
}
|
|
|
|
/// Report whether the limiter currently has capacity without consuming a
|
|
/// token, rolling the window over first so an elapsed window reports
|
|
/// capacity again.
|
|
///
|
|
/// Used by `poll_ready` so an exhausted limiter reports readiness instead
|
|
/// of blocking on inner-service backpressure: `call` can then return
|
|
/// `RESOURCE_EXHAUSTED` immediately rather than waiting for the inner gRPC
|
|
/// service to become ready.
|
|
fn has_capacity(&self) -> bool {
|
|
let now = Instant::now();
|
|
let mut state = self
|
|
.state
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
if now.duration_since(state.window_started) >= self.window {
|
|
state.window_started = now;
|
|
state.remaining = self.requests;
|
|
}
|
|
state.remaining > 0
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct GrpcRateLimitService<S> {
|
|
inner: S,
|
|
limiter: Option<GrpcRateLimiter>,
|
|
/// Set by `poll_ready` when it reports synthetic readiness for an
|
|
/// exhausted limiter without polling the inner service. The paired `call`
|
|
/// must then reject with `RESOURCE_EXHAUSTED` instead of forwarding to an
|
|
/// inner service that never reported readiness — even if the rate-limit
|
|
/// window rolls over in between. Reset whenever `poll_ready` defers to the
|
|
/// inner service.
|
|
rate_limited: bool,
|
|
}
|
|
|
|
impl<S> GrpcRateLimitService<S> {
|
|
fn new(inner: S, limiter: Option<GrpcRateLimiter>) -> Self {
|
|
Self {
|
|
inner,
|
|
limiter,
|
|
rate_limited: false,
|
|
}
|
|
}
|
|
}
|
|
|
|
impl<S, B> tower::Service<Request<B>> for GrpcRateLimitService<S>
|
|
where
|
|
S: tower::Service<Request<B>, Response = Response<tonic::body::Body>>,
|
|
S::Future: Send + 'static,
|
|
B: Send + 'static,
|
|
{
|
|
type Response = S::Response;
|
|
type Error = S::Error;
|
|
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
|
|
|
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
|
// When the limiter is exhausted, report ready so `call` can return
|
|
// RESOURCE_EXHAUSTED immediately. Delegating to the inner service here
|
|
// would make rate-limited requests wait on inner backpressure (a
|
|
// pending inner `poll_ready`) before they are rejected. The check is
|
|
// non-consuming: the token is only consumed in `call` via `allow`.
|
|
//
|
|
// Crucially, this path does NOT poll the inner service, so the inner
|
|
// service has not reported readiness. Record that decision so the
|
|
// paired `call` rejects rather than forwarding to a service that never
|
|
// became ready — even if the rate-limit window rolls over in between.
|
|
if self
|
|
.limiter
|
|
.as_ref()
|
|
.is_some_and(|limiter| !limiter.has_capacity())
|
|
{
|
|
self.rate_limited = true;
|
|
return Poll::Ready(Ok(()));
|
|
}
|
|
self.rate_limited = false;
|
|
self.inner.poll_ready(cx)
|
|
}
|
|
|
|
fn call(&mut self, req: Request<B>) -> Self::Future {
|
|
// If `poll_ready` short-circuited an exhausted limiter, it never polled
|
|
// the inner service to readiness. Honor that decision regardless of the
|
|
// limiter's current state (the window may have rolled over since): the
|
|
// Tower contract forbids forwarding to an inner service that did not
|
|
// report readiness.
|
|
if std::mem::take(&mut self.rate_limited) {
|
|
let response =
|
|
tonic::Status::resource_exhausted("gRPC rate limit exceeded").into_http();
|
|
return Box::pin(async move { Ok(response) });
|
|
}
|
|
if self
|
|
.limiter
|
|
.as_ref()
|
|
.is_some_and(|limiter| !limiter.allow())
|
|
{
|
|
let response =
|
|
tonic::Status::resource_exhausted("gRPC rate limit exceeded").into_http();
|
|
return Box::pin(async move { Ok(response) });
|
|
}
|
|
let future = self.inner.call(req);
|
|
Box::pin(future)
|
|
}
|
|
}
|
|
|
|
/// Combined gRPC service that routes between `OpenShell` and reflection.
|
|
#[derive(Clone)]
|
|
pub struct GrpcRouter<N, R> {
|
|
openshell: N,
|
|
reflection: R,
|
|
}
|
|
|
|
impl<N, R> GrpcRouter<N, R> {
|
|
fn new(openshell: N, reflection: R) -> Self {
|
|
Self {
|
|
openshell,
|
|
reflection,
|
|
}
|
|
}
|
|
}
|
|
|
|
pub const REFLECTION_PATH_PREFIX: &str = "/grpc.reflection.v1.";
|
|
|
|
impl<N, R, B> tower::Service<Request<B>> for GrpcRouter<N, R>
|
|
where
|
|
N: tower::Service<Request<B>> + Clone + Send + 'static,
|
|
N::Response: Send,
|
|
N::Future: Send,
|
|
N::Error: Send,
|
|
R: tower::Service<Request<B>, Response = N::Response, Error = N::Error>
|
|
+ Clone
|
|
+ Send
|
|
+ 'static,
|
|
R::Future: Send,
|
|
B: Send + 'static,
|
|
{
|
|
type Response = N::Response;
|
|
type Error = N::Error;
|
|
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
|
|
|
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
|
Poll::Ready(Ok(()))
|
|
}
|
|
|
|
fn call(&mut self, req: Request<B>) -> Self::Future {
|
|
if req.uri().path().starts_with(REFLECTION_PATH_PREFIX) {
|
|
let mut svc = self.reflection.clone();
|
|
Box::pin(async move { svc.ready().await?.call(req).await })
|
|
} else {
|
|
let mut svc = self.openshell.clone();
|
|
Box::pin(async move { svc.ready().await?.call(req).await })
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Assemble the authenticator chain for the gateway.
|
|
///
|
|
/// Chain order (first-match-wins):
|
|
/// 1. `PeerServiceAccountAuthenticator` (path-scoped to peer RPCs)
|
|
/// — validates gateway replica projected `ServiceAccount` tokens with
|
|
/// `TokenReview` for internal peer relay calls. No-op on every other path.
|
|
/// 2. `ComputeDriverAuthenticator` (path-scoped to `IssueSandboxToken`)
|
|
/// — delegates a driver-native credential and receives a sandbox identity
|
|
/// so the handler can mint a gateway JWT. No-op on every other path.
|
|
/// 3. `SandboxSessionJwtAuthenticator` — validates generation-bound gateway
|
|
/// JWTs against the durable sandbox identity. Untyped legacy sandbox JWTs
|
|
/// are not admitted.
|
|
/// 4. `OidcAuthenticator` — validates user Bearer tokens against the
|
|
/// configured OIDC issuer. Returns `Unauthenticated` for missing
|
|
/// Bearer headers so non-OIDC clients can't sneak through.
|
|
///
|
|
/// Once sandbox authentication is configured, callers must present an
|
|
/// explicit credential for authenticated gRPC methods. Missing bearer auth
|
|
/// is promoted to an mTLS user only when `mtls_auth.enabled` is configured
|
|
/// and the connection presents a verified client certificate, or to an unsafe
|
|
/// local developer user when `auth.allow_unauthenticated_users` is explicitly
|
|
/// enabled.
|
|
///
|
|
/// When neither OIDC nor sandbox credentials are configured (a barebones
|
|
/// dev gateway), the chain is left as `None` so the router short-circuits
|
|
/// to pass-through unless mTLS or local unauthenticated users are enabled.
|
|
fn build_authenticator_chain(state: &ServerState) -> Option<AuthenticatorChain> {
|
|
let mut authenticators: Vec<Arc<dyn crate::auth::authenticator::Authenticator>> = Vec::new();
|
|
if let Some(peer) = state.peer_authenticator.clone() {
|
|
authenticators.push(peer);
|
|
}
|
|
if let Some(driver) = state.compute_driver_authenticator.clone() {
|
|
authenticators.push(driver);
|
|
}
|
|
if let Some(authority) = state.sandbox_session_jwt_authority.clone() {
|
|
authenticators.push(Arc::new(
|
|
crate::auth::sandbox_jwt::SandboxSessionJwtAuthenticator::new(
|
|
authority,
|
|
state.store.clone(),
|
|
),
|
|
));
|
|
}
|
|
if let Some(cache) = state.oidc_cache.clone() {
|
|
authenticators.push(Arc::new(OidcAuthenticator::new(cache)));
|
|
}
|
|
if authenticators.is_empty() {
|
|
return None;
|
|
}
|
|
Some(AuthenticatorChain::new(authenticators))
|
|
}
|
|
|
|
/// gRPC router wrapper that runs the [`AuthenticatorChain`] and inserts the
|
|
/// resulting [`Principal`] into the request's extensions.
|
|
///
|
|
/// Behavior:
|
|
/// - Strip any external `x-openshell-auth-source` marker first (so callers
|
|
/// cannot spoof a sandbox identity).
|
|
/// - Health probes / reflection bypass the chain entirely.
|
|
/// - When no chain is configured (OIDC not configured), forward without
|
|
/// authentication — preserves today's pass-through behavior.
|
|
/// - Otherwise, run the chain. The first match produces a `Principal`.
|
|
/// `Principal::User` is gated by the RBAC `AuthzPolicy`.
|
|
/// `Principal::Sandbox` is gated by a supervisor-method allowlist, then
|
|
/// handlers enforce same-sandbox scope on request bodies.
|
|
#[derive(Clone)]
|
|
pub struct AuthGrpcRouter<S> {
|
|
inner: S,
|
|
authenticator_chain: Option<AuthenticatorChain>,
|
|
authz_policy: Option<AuthzPolicy>,
|
|
/// mTLS peer identity extracted from the TLS handshake.
|
|
peer_identity: Option<Identity>,
|
|
peer_certificate_sha256: Option<String>,
|
|
mtls_auth_enabled: bool,
|
|
allow_unauthenticated_users: bool,
|
|
operator: Option<OperatorPrincipal>,
|
|
connection_source: Option<ConnectionSource>,
|
|
logons: audit::Logons,
|
|
}
|
|
|
|
impl<S> AuthGrpcRouter<S> {
|
|
#[cfg(test)]
|
|
fn new(
|
|
inner: S,
|
|
authenticator_chain: Option<AuthenticatorChain>,
|
|
authz_policy: Option<AuthzPolicy>,
|
|
) -> Self {
|
|
Self::with_peer_identity(inner, authenticator_chain, authz_policy, None, false, false)
|
|
}
|
|
|
|
fn with_peer_identity(
|
|
inner: S,
|
|
authenticator_chain: Option<AuthenticatorChain>,
|
|
authz_policy: Option<AuthzPolicy>,
|
|
peer_identity: Option<Identity>,
|
|
mtls_auth_enabled: bool,
|
|
allow_unauthenticated_users: bool,
|
|
) -> Self {
|
|
Self {
|
|
inner,
|
|
authenticator_chain,
|
|
authz_policy,
|
|
peer_identity,
|
|
peer_certificate_sha256: None,
|
|
mtls_auth_enabled,
|
|
allow_unauthenticated_users,
|
|
operator: None,
|
|
connection_source: None,
|
|
logons: audit::Logons::default(),
|
|
}
|
|
}
|
|
|
|
fn with_operator(mut self, operator: Option<OperatorPrincipal>) -> Self {
|
|
self.operator = operator;
|
|
self
|
|
}
|
|
|
|
fn with_connection_source(mut self, source: ConnectionSource) -> Self {
|
|
self.connection_source = Some(source);
|
|
self
|
|
}
|
|
|
|
/// The fingerprint of the verified client certificate that
|
|
/// authenticated `peer_identity`.
|
|
fn with_peer_certificate(mut self, certificate_sha256: Option<String>) -> Self {
|
|
self.peer_certificate_sha256 = certificate_sha256;
|
|
self
|
|
}
|
|
|
|
/// Share logon records with the gateway's other connections.
|
|
fn with_logons(mut self, logons: audit::Logons) -> Self {
|
|
self.logons = logons;
|
|
self
|
|
}
|
|
}
|
|
|
|
fn unauthenticated_dev_user_principal() -> Principal {
|
|
Principal::User(UserPrincipal {
|
|
identity: Identity {
|
|
subject: "unauthenticated-local-dev".to_string(),
|
|
display_name: Some("Unauthenticated Local Dev".to_string()),
|
|
roles: vec!["openshell-user".to_string(), "openshell-admin".to_string()],
|
|
scopes: vec!["openshell:all".to_string()],
|
|
provider: crate::auth::identity::IdentityProvider::LocalDev,
|
|
},
|
|
})
|
|
}
|
|
|
|
fn status_response(status: tonic::Status) -> Response<tonic::body::Body> {
|
|
status.into_http()
|
|
}
|
|
|
|
impl<S, B> tower::Service<Request<B>> for AuthGrpcRouter<S>
|
|
where
|
|
S: tower::Service<Request<B>, Response = Response<tonic::body::Body>> + Clone + Send + 'static,
|
|
S::Future: Send,
|
|
S::Error: Send + Into<Box<dyn std::error::Error + Send + Sync>>,
|
|
B: Send + 'static,
|
|
{
|
|
type Response = S::Response;
|
|
type Error = S::Error;
|
|
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
|
|
|
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
|
Poll::Ready(Ok(()))
|
|
}
|
|
|
|
fn call(&mut self, req: Request<B>) -> Self::Future {
|
|
let chain = self.authenticator_chain.clone();
|
|
let authz_policy = self.authz_policy.clone();
|
|
let peer_identity = self.peer_identity.clone();
|
|
let mtls_auth_enabled = self.mtls_auth_enabled;
|
|
let allow_unauthenticated_users = self.allow_unauthenticated_users;
|
|
let operator = self.operator.clone();
|
|
let connection_source = self.connection_source;
|
|
let logons = self.logons.clone();
|
|
let peer_certificate_sha256 = self.peer_certificate_sha256.clone();
|
|
let mut inner = self.inner.clone();
|
|
|
|
Box::pin(async move {
|
|
let mut req = req;
|
|
if let Some(source) = connection_source {
|
|
req.extensions_mut().insert(source);
|
|
}
|
|
|
|
let path = req.uri().path().to_string();
|
|
let origin = |headers: &http::HeaderMap| RequestOrigin::new(connection_source, headers);
|
|
|
|
// Health probes and reflection — truly unauthenticated.
|
|
if oidc::is_unauthenticated_method(&path) {
|
|
return inner.ready().await?.call(req).await;
|
|
}
|
|
|
|
let principal = if let Some(operator) = operator {
|
|
if req.headers().contains_key(http::header::AUTHORIZATION) {
|
|
audit::emit_authentication_failure(
|
|
&path,
|
|
&origin(req.headers()),
|
|
Some(AuthMechanism::Mtls),
|
|
AuthFailureReason::ConflictingCredentials,
|
|
);
|
|
return Ok(status_response(tonic::Status::unauthenticated(
|
|
"operator certificates must not be combined with authorization headers",
|
|
)));
|
|
}
|
|
Principal::Operator(operator)
|
|
} else if let Some(chain) = chain {
|
|
match chain.authenticate(req.headers(), &path).await {
|
|
Ok(Authentication::Accepted(p)) => p,
|
|
Ok(outcome) => match (mtls_auth_enabled, peer_identity) {
|
|
(true, Some(identity)) => Principal::User(UserPrincipal { identity }),
|
|
_ if allow_unauthenticated_users => unauthenticated_dev_user_principal(),
|
|
_ => {
|
|
let (mechanism, reason) = match outcome {
|
|
Authentication::Declined(decline) => {
|
|
(Some(decline.mechanism), decline.reason)
|
|
}
|
|
_ => (None, AuthFailureReason::MissingCredential),
|
|
};
|
|
audit::emit_authentication_failure(
|
|
&path,
|
|
&origin(req.headers()),
|
|
mechanism,
|
|
reason,
|
|
);
|
|
return Ok(status_response(tonic::Status::unauthenticated(
|
|
"missing authorization header",
|
|
)));
|
|
}
|
|
},
|
|
Err(rejection) => {
|
|
audit::emit_authentication_failure(
|
|
&path,
|
|
&origin(req.headers()),
|
|
Some(rejection.mechanism),
|
|
rejection.reason,
|
|
);
|
|
return Ok(status_response(rejection.status));
|
|
}
|
|
}
|
|
} else if mtls_auth_enabled {
|
|
let Some(identity) = peer_identity else {
|
|
audit::emit_authentication_failure(
|
|
&path,
|
|
&origin(req.headers()),
|
|
Some(AuthMechanism::Mtls),
|
|
AuthFailureReason::MissingCredential,
|
|
);
|
|
return Ok(status_response(tonic::Status::unauthenticated(
|
|
"missing client certificate",
|
|
)));
|
|
};
|
|
Principal::User(UserPrincipal { identity })
|
|
} else if allow_unauthenticated_users {
|
|
unauthenticated_dev_user_principal()
|
|
} else {
|
|
// No auth configured — dev / fronting-proxy deployments.
|
|
// Inject a local-dev principal so downstream handlers that
|
|
// call extract_principal() always find one.
|
|
unauthenticated_dev_user_principal()
|
|
};
|
|
let origin = origin(req.headers());
|
|
audit::emit_logon(
|
|
&logons,
|
|
&path,
|
|
&origin,
|
|
&principal,
|
|
req.headers(),
|
|
peer_certificate_sha256.as_deref(),
|
|
);
|
|
|
|
let method_denied = |message: &str| {
|
|
let denial = if crate::auth::method_authz::lookup(&path).is_some() {
|
|
AuthzDenial::MethodNotPermitted
|
|
} else {
|
|
AuthzDenial::UnknownMethod
|
|
};
|
|
Some((denial, tonic::Status::permission_denied(message)))
|
|
};
|
|
let denial = match principal {
|
|
Principal::Operator(_)
|
|
if !crate::auth::method_authz::is_operator_callable(&path) =>
|
|
{
|
|
method_denied("operators may not call supervisor, peer, or unknown methods")
|
|
}
|
|
Principal::User(_) if !crate::auth::method_authz::is_user_callable(&path) => {
|
|
method_denied("this method requires a sandbox principal")
|
|
}
|
|
Principal::User(ref user) => authz_policy
|
|
.as_ref()
|
|
.and_then(|policy| policy.check(&user.identity, &path).err())
|
|
.map(|rbac| (rbac.denial, rbac.status)),
|
|
Principal::Sandbox(_)
|
|
if !crate::auth::sandbox_methods::is_sandbox_callable(&path) =>
|
|
{
|
|
method_denied("sandbox principals may not call this method")
|
|
}
|
|
Principal::Peer(_) if !crate::auth::method_authz::is_peer_callable(&path) => {
|
|
method_denied("gateway peer principals may not call this method")
|
|
}
|
|
Principal::Anonymous => {
|
|
return Ok(status_response(tonic::Status::unauthenticated(
|
|
"anonymous callers may not call authenticated methods",
|
|
)));
|
|
}
|
|
Principal::Operator(_) | Principal::Sandbox(_) | Principal::Peer(_) => None,
|
|
};
|
|
if let Some((denial, status)) = denial {
|
|
audit::emit_authorization_denial_for(&path, &origin, &principal, denial);
|
|
return Ok(status_response(status));
|
|
}
|
|
|
|
req.extensions_mut().insert(principal);
|
|
let call = inner.ready().await?.call(req);
|
|
audit::with_request(path, origin, call).await
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Service that multiplexes between gRPC and HTTP.
|
|
#[derive(Clone)]
|
|
pub struct MultiplexedService<G, H> {
|
|
grpc: G,
|
|
http: H,
|
|
}
|
|
|
|
impl<G, H> MultiplexedService<G, H> {
|
|
/// Create a new multiplexed service from gRPC and HTTP services.
|
|
#[must_use]
|
|
pub fn new(grpc: G, http: H) -> Self {
|
|
Self { grpc, http }
|
|
}
|
|
}
|
|
|
|
impl<G, H, GBody, HBody> hyper::service::Service<Request<Incoming>> for MultiplexedService<G, H>
|
|
where
|
|
G: tower::Service<Request<BoxBody>, Response = Response<GBody>> + Clone + Send + 'static,
|
|
G::Future: Send,
|
|
G::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
|
GBody: Body<Data = Bytes> + Send + 'static,
|
|
GBody::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
|
H: tower::Service<Request<BoxBody>, Response = Response<HBody>> + Clone + Send + 'static,
|
|
H::Future: Send,
|
|
H::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
|
HBody: Body<Data = Bytes> + Send + 'static,
|
|
HBody::Error: Into<Box<dyn std::error::Error + Send + Sync>>,
|
|
{
|
|
type Response = Response<BoxBody>;
|
|
type Error = Box<dyn std::error::Error + Send + Sync>;
|
|
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
|
|
|
fn call(&self, req: Request<Incoming>) -> Self::Future {
|
|
let is_grpc = req
|
|
.headers()
|
|
.get("content-type")
|
|
.is_some_and(|v| v.as_bytes().starts_with(b"application/grpc"));
|
|
|
|
if is_grpc {
|
|
let method = grpc_method_from_path(req.uri().path());
|
|
let start = Instant::now();
|
|
let mut grpc = self.grpc.clone();
|
|
Box::pin(async move {
|
|
let (parts, body) = req.into_parts();
|
|
let body = body.map_err(Into::into).boxed_unsync();
|
|
let req = Request::from_parts(parts, BoxBody(body));
|
|
|
|
let res = grpc
|
|
.ready()
|
|
.await
|
|
.map_err(Into::into)?
|
|
.call(req)
|
|
.await
|
|
.map_err(Into::into)?;
|
|
|
|
let code = grpc_status_from_response(&res);
|
|
let elapsed = start.elapsed().as_secs_f64();
|
|
counter!("openshell_server_grpc_requests_total", "method" => method.clone(), "code" => code.clone()).increment(1);
|
|
histogram!("openshell_server_grpc_request_duration_seconds", "method" => method, "code" => code).record(elapsed);
|
|
|
|
let (parts, body) = res.into_parts();
|
|
let body = body.map_err(Into::into).boxed_unsync();
|
|
Ok(Response::from_parts(parts, BoxBody(body)))
|
|
})
|
|
} else {
|
|
let path = normalize_http_path(req.uri().path());
|
|
let start = Instant::now();
|
|
let mut http = self.http.clone();
|
|
Box::pin(async move {
|
|
let (parts, body) = req.into_parts();
|
|
let body = body.map_err(Into::into).boxed_unsync();
|
|
let req = Request::from_parts(parts, BoxBody(body));
|
|
|
|
let res = http
|
|
.ready()
|
|
.await
|
|
.map_err(Into::into)?
|
|
.call(req)
|
|
.await
|
|
.map_err(Into::into)?;
|
|
|
|
let status = res.status().as_u16().to_string();
|
|
let elapsed = start.elapsed().as_secs_f64();
|
|
counter!("openshell_server_http_requests_total", "path" => path, "status" => status.clone()).increment(1);
|
|
histogram!("openshell_server_http_request_duration_seconds", "path" => path, "status" => status).record(elapsed);
|
|
|
|
let (parts, body) = res.into_parts();
|
|
let body = body.map_err(Into::into).boxed_unsync();
|
|
Ok(Response::from_parts(parts, BoxBody(body)))
|
|
})
|
|
}
|
|
}
|
|
}
|
|
|
|
fn grpc_method_from_path(path: &str) -> String {
|
|
path.rsplit('/').next().unwrap_or(path).to_string()
|
|
}
|
|
|
|
/// Name for the exported `OpenTelemetry` span, per the `OTel` semantic
|
|
/// conventions: `$service/$method` for RPCs and the method for plain HTTP.
|
|
///
|
|
/// The gateway cannot determine route templates for proxied sandbox
|
|
/// applications, so including the literal path would create high-cardinality
|
|
/// operation names. The path remains available as a span attribute.
|
|
///
|
|
/// The `tracing` callsite name is the constant `"request"` because `tracing`
|
|
/// requires `'static` span names, so the per-request name is carried in the
|
|
/// `otel.name` field instead.
|
|
fn otel_span_name(method: &http::Method, path: &str) -> String {
|
|
grpc_service_method(path).map_or_else(
|
|
|| method.to_string(),
|
|
|(service, rpc_method)| format!("{service}/{rpc_method}"),
|
|
)
|
|
}
|
|
|
|
/// Split a gRPC path into its service and method.
|
|
///
|
|
/// A gRPC path is exactly "/package.Service/Method". Anything else — a health
|
|
/// check, /metrics, a sandbox service URL — is plain HTTP.
|
|
fn grpc_service_method(path: &str) -> Option<(&str, &str)> {
|
|
let mut segments = path.strip_prefix('/')?.split('/');
|
|
let service = segments.next()?;
|
|
let method = segments.next()?;
|
|
if segments.next().is_some() || !service.contains('.') || method.is_empty() {
|
|
return None;
|
|
}
|
|
Some((service, method))
|
|
}
|
|
|
|
fn grpc_status_from_response<B>(res: &Response<B>) -> String {
|
|
res.headers()
|
|
.get("grpc-status")
|
|
.and_then(|v| v.to_str().ok())
|
|
.map_or_else(|| "0".to_string(), ToString::to_string)
|
|
}
|
|
|
|
fn grpc_status_from_response_and_trailers<B>(
|
|
res: &Response<B>,
|
|
trailers: Option<&http::HeaderMap>,
|
|
) -> String {
|
|
trailers
|
|
.and_then(|trailers| trailers.get("grpc-status"))
|
|
.or_else(|| res.headers().get("grpc-status"))
|
|
.and_then(|value| value.to_str().ok())
|
|
.map_or_else(|| "0".to_string(), ToString::to_string)
|
|
}
|
|
|
|
fn normalize_http_path(path: &str) -> &'static str {
|
|
match path {
|
|
p if p.starts_with("/_ws_tunnel") => "/_ws_tunnel",
|
|
p if p.starts_with("/auth/") => "/auth",
|
|
_ => "unknown",
|
|
}
|
|
}
|
|
|
|
/// Extract an [`Identity`] from the peer certificates presented during a TLS
|
|
/// handshake. Returns `None` if no client certificate was presented.
|
|
pub fn extract_peer_identity<S>(tls_stream: &tokio_rustls::server::TlsStream<S>) -> Option<Identity>
|
|
where
|
|
S: AsyncRead + AsyncWrite + Unpin,
|
|
{
|
|
use crate::auth::identity::IdentityProvider;
|
|
use x509_parser::prelude::*;
|
|
|
|
let (_, server_conn) = tls_stream.get_ref();
|
|
let certs = server_conn.peer_certificates()?;
|
|
let first = certs.first()?;
|
|
|
|
let (_, cert) = X509Certificate::from_der(first.as_ref()).ok()?;
|
|
let subject = cert.subject();
|
|
|
|
let cn = subject
|
|
.iter_common_name()
|
|
.next()
|
|
.and_then(|attr| attr.as_str().ok())
|
|
.unwrap_or("unknown")
|
|
.to_string();
|
|
|
|
let roles: Vec<String> = subject
|
|
.iter_organizational_unit()
|
|
.filter_map(|attr| attr.as_str().ok().map(String::from))
|
|
.collect();
|
|
|
|
Some(Identity {
|
|
subject: cn.clone(),
|
|
display_name: Some(cn),
|
|
roles,
|
|
scopes: Vec::new(),
|
|
provider: IdentityProvider::Mtls,
|
|
})
|
|
}
|
|
|
|
/// Fingerprint of the leaf certificate verified during the TLS handshake.
|
|
pub fn extract_peer_certificate_sha256<S>(
|
|
stream: &tokio_rustls::server::TlsStream<S>,
|
|
) -> Result<Option<String>, openshell_crypto::CryptoError> {
|
|
let Some(certificate) = stream
|
|
.get_ref()
|
|
.1
|
|
.peer_certificates()
|
|
.and_then(|certs| certs.first())
|
|
else {
|
|
return Ok(None);
|
|
};
|
|
openshell_crypto::sha256(certificate.as_ref()).map(|digest| Some(hex::encode(digest)))
|
|
}
|
|
|
|
/// Boxed body type for uniform handling.
|
|
pub struct BoxBody(
|
|
http_body_util::combinators::UnsyncBoxBody<Bytes, Box<dyn std::error::Error + Send + Sync>>,
|
|
);
|
|
|
|
impl Body for BoxBody {
|
|
type Data = Bytes;
|
|
type Error = Box<dyn std::error::Error + Send + Sync>;
|
|
|
|
fn poll_frame(
|
|
mut self: Pin<&mut Self>,
|
|
cx: &mut Context<'_>,
|
|
) -> Poll<Option<Result<http_body::Frame<Self::Data>, Self::Error>>> {
|
|
Pin::new(&mut self.0).poll_frame(cx)
|
|
}
|
|
|
|
fn is_end_stream(&self) -> bool {
|
|
self.0.is_end_stream()
|
|
}
|
|
|
|
fn size_hint(&self) -> http_body::SizeHint {
|
|
self.0.size_hint()
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use bytes::Bytes;
|
|
use http::StatusCode;
|
|
use http_body_util::Empty;
|
|
use openshell_core::GatewayInterceptorConfig;
|
|
use openshell_core::proto::CreateSandboxRequest;
|
|
use openshell_core::proto::gateway_interceptor::v1::{
|
|
DescribeRequest, GatewayInterceptorPhase, InterceptorBinding, InterceptorEvaluation,
|
|
InterceptorManifest, InterceptorResult, InterceptorSelector, ProviderProfileSnapshot,
|
|
ProviderProfileSnapshotRequest,
|
|
gateway_interceptor_server::{GatewayInterceptor, GatewayInterceptorServer},
|
|
};
|
|
use std::convert::Infallible;
|
|
use std::sync::Mutex;
|
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
|
use tokio_stream::wrappers::TcpListenerStream;
|
|
use tower::Service;
|
|
|
|
#[derive(Clone)]
|
|
struct PostCommitTestInterceptor;
|
|
|
|
#[tonic::async_trait]
|
|
impl GatewayInterceptor for PostCommitTestInterceptor {
|
|
async fn describe(
|
|
&self,
|
|
_request: tonic::Request<DescribeRequest>,
|
|
) -> Result<tonic::Response<InterceptorManifest>, tonic::Status> {
|
|
Ok(tonic::Response::new(InterceptorManifest {
|
|
name: "post-commit-test".to_string(),
|
|
failure_policy: "fail_open".to_string(),
|
|
bindings: vec![InterceptorBinding {
|
|
id: "audit-create-sandbox".to_string(),
|
|
selector: Some(InterceptorSelector {
|
|
rpc: "openshell.v1.OpenShell/CreateSandbox".to_string(),
|
|
service: String::new(),
|
|
method: String::new(),
|
|
}),
|
|
phases: vec![GatewayInterceptorPhase::PostCommit as i32],
|
|
failure_policy: "fail_open".to_string(),
|
|
}],
|
|
provider_profiles: false,
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
openshell_core::extension_protocol::ExtensionFamily::GatewayInterceptor,
|
|
"openshell/post-commit-test",
|
|
"test",
|
|
[],
|
|
)),
|
|
}))
|
|
}
|
|
|
|
async fn evaluate(
|
|
&self,
|
|
_request: tonic::Request<InterceptorEvaluation>,
|
|
) -> Result<tonic::Response<InterceptorResult>, tonic::Status> {
|
|
Ok(tonic::Response::new(InterceptorResult {
|
|
allowed: true,
|
|
..InterceptorResult::default()
|
|
}))
|
|
}
|
|
|
|
async fn snapshot_provider_profiles(
|
|
&self,
|
|
_request: tonic::Request<ProviderProfileSnapshotRequest>,
|
|
) -> Result<tonic::Response<ProviderProfileSnapshot>, tonic::Status> {
|
|
Err(tonic::Status::unimplemented("not a profile source"))
|
|
}
|
|
}
|
|
|
|
fn grpc_frame(message: &[u8]) -> Bytes {
|
|
let mut frame = Vec::with_capacity(5 + message.len());
|
|
frame.push(0);
|
|
frame.extend_from_slice(&u32::try_from(message.len()).unwrap().to_be_bytes());
|
|
frame.extend_from_slice(message);
|
|
Bytes::from(frame)
|
|
}
|
|
|
|
async fn post_commit_test_runtime() -> (GatewayInterceptorRuntime, tokio::task::JoinHandle<()>)
|
|
{
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let task = tokio::spawn(async move {
|
|
tonic::transport::Server::builder()
|
|
.add_service(GatewayInterceptorServer::new(PostCommitTestInterceptor))
|
|
.serve_with_incoming(TcpListenerStream::new(listener))
|
|
.await
|
|
.unwrap();
|
|
});
|
|
let runtime = openshell_gateway_interceptors::initialize(vec![GatewayInterceptorConfig {
|
|
name: "post-commit-test".to_string(),
|
|
grpc_endpoint: format!("http://{address}"),
|
|
..GatewayInterceptorConfig::default()
|
|
}])
|
|
.await
|
|
.unwrap()
|
|
.unwrap();
|
|
(runtime, task)
|
|
}
|
|
|
|
#[test]
|
|
fn uuid_request_id_generates_valid_uuid() {
|
|
let mut maker = UuidRequestId;
|
|
let req = Request::builder().body(()).unwrap();
|
|
let id = maker.make_request_id(&req).expect("should produce an ID");
|
|
let value = id.header_value().to_str().unwrap();
|
|
uuid::Uuid::parse_str(value).expect("should be a valid UUID");
|
|
}
|
|
|
|
#[test]
|
|
fn uuid_request_id_generates_unique_ids() {
|
|
let mut maker = UuidRequestId;
|
|
let req = Request::builder().body(()).unwrap();
|
|
let id1 = maker.make_request_id(&req).unwrap();
|
|
let id2 = maker.make_request_id(&req).unwrap();
|
|
assert_ne!(id1.header_value(), id2.header_value());
|
|
}
|
|
|
|
async fn test_health_store() -> Arc<crate::Store> {
|
|
Arc::new(
|
|
crate::Store::connect("sqlite::memory:")
|
|
.await
|
|
.expect("connect in-memory sqlite store for tests"),
|
|
)
|
|
}
|
|
|
|
async fn start_http_server_with_middleware() -> std::net::SocketAddr {
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let addr = listener.local_addr().unwrap();
|
|
|
|
let http_service = crate::http::health_router(test_health_store().await);
|
|
let http_service = request_id_middleware!(http_service);
|
|
|
|
let service = MultiplexedService::new(http_service.clone(), http_service);
|
|
|
|
tokio::spawn(async move {
|
|
loop {
|
|
let Ok((stream, _)) = listener.accept().await else {
|
|
continue;
|
|
};
|
|
let svc = service.clone();
|
|
tokio::spawn(async move {
|
|
let _ = Builder::new(TokioExecutor::new())
|
|
.serve_connection(TokioIo::new(stream), svc)
|
|
.await;
|
|
});
|
|
}
|
|
});
|
|
|
|
addr
|
|
}
|
|
|
|
async fn http1_request(
|
|
addr: std::net::SocketAddr,
|
|
method: &str,
|
|
path: &str,
|
|
headers: &[(&str, &str)],
|
|
) -> Response<Incoming> {
|
|
let stream = tokio::net::TcpStream::connect(addr).await.unwrap();
|
|
let (mut sender, conn) = hyper::client::conn::http1::Builder::new()
|
|
.handshake(TokioIo::new(stream))
|
|
.await
|
|
.unwrap();
|
|
tokio::spawn(async move {
|
|
let _ = conn.await;
|
|
});
|
|
|
|
let mut builder = Request::builder()
|
|
.method(method)
|
|
.uri(format!("http://{addr}{path}"));
|
|
for (k, v) in headers {
|
|
builder = builder.header(*k, *v);
|
|
}
|
|
let req = builder.body(Empty::<Bytes>::new()).unwrap();
|
|
sender.send_request(req).await.unwrap()
|
|
}
|
|
|
|
async fn http1_get(
|
|
addr: std::net::SocketAddr,
|
|
path: &str,
|
|
headers: &[(&str, &str)],
|
|
) -> Response<Incoming> {
|
|
http1_request(addr, "GET", path, headers).await
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn intercepted_grpc_body_collection_rejects_oversized_body() {
|
|
let oversized = Bytes::from(vec![0_u8; MAX_INTERCEPTED_GRPC_BODY_SIZE + 1]);
|
|
let status = collect_intercepted_grpc_body(boxed_body_from_bytes(oversized))
|
|
.await
|
|
.expect_err("oversized body should be rejected");
|
|
|
|
assert_eq!(status.code(), tonic::Code::ResourceExhausted);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn update_provider_interception_hydrates_identity_from_trusted_state() {
|
|
let state = crate::grpc::test_support::test_server_state().await;
|
|
let existing = Provider {
|
|
metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta {
|
|
id: "provider-id".to_string(),
|
|
name: "managed-provider".to_string(),
|
|
workspace: "default".to_string(),
|
|
..Default::default()
|
|
}),
|
|
r#type: "agent-tool-gateway".to_string(),
|
|
profile_workspace: "default".to_string(),
|
|
..Default::default()
|
|
};
|
|
state.store.put_message(&existing).await.unwrap();
|
|
|
|
let request = UpdateProviderRequest {
|
|
provider: Some(Provider {
|
|
metadata: Some(openshell_core::proto::datamodel::v1::ObjectMeta {
|
|
name: "managed-provider".to_string(),
|
|
workspace: "default".to_string(),
|
|
..Default::default()
|
|
}),
|
|
credentials: std::collections::HashMap::from([(
|
|
"TOKEN".to_string(),
|
|
"rotated".to_string(),
|
|
)]),
|
|
..Default::default()
|
|
}),
|
|
workspace_scope: Some(openshell_core::proto::workspace_selector(
|
|
"default".to_string(),
|
|
)),
|
|
..Default::default()
|
|
};
|
|
let authed = crate::grpc::test_support::authed_request(());
|
|
let principal = authed.extensions().get::<Principal>().unwrap();
|
|
|
|
let hydrated = hydrate_update_provider_identity(
|
|
UPDATE_PROVIDER_PATH,
|
|
grpc_frame(&request.encode_to_vec()),
|
|
state.as_ref(),
|
|
Some(principal),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
let hydrated = decode_unary_grpc_message::<UpdateProviderRequest>(&hydrated).unwrap();
|
|
let provider = hydrated.provider.unwrap();
|
|
|
|
assert_eq!(provider.r#type, "agent-tool-gateway");
|
|
assert_eq!(provider.profile_workspace, "default");
|
|
assert_eq!(provider.credentials.get("TOKEN").unwrap(), "rotated");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn intercepted_grpc_response_preserves_body_and_trailers() {
|
|
let bytes = Bytes::from_static(b"committed-response");
|
|
let mut trailers = http::HeaderMap::new();
|
|
trailers.insert("grpc-status", HeaderValue::from_static("0"));
|
|
let response = Response::new(tonic_body_from_bytes_and_trailers(
|
|
bytes.clone(),
|
|
Some(trailers.clone()),
|
|
));
|
|
|
|
let (response, observation) = observe_intercepted_grpc_response(response).await;
|
|
let (observed, observed_trailers) = observation.unwrap();
|
|
|
|
assert_eq!(observed, bytes);
|
|
assert_eq!(observed_trailers.as_ref(), Some(&trailers));
|
|
assert_eq!(
|
|
grpc_status_from_response_and_trailers(&response, observed_trailers.as_ref()),
|
|
"0"
|
|
);
|
|
|
|
let collected = response.into_body().collect().await.unwrap();
|
|
assert_eq!(collected.trailers(), Some(&trailers));
|
|
assert_eq!(collected.to_bytes(), bytes);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn intercepted_grpc_response_preserves_body_error() {
|
|
let bytes = Bytes::from_static(b"committed-response-prefix");
|
|
let response = Response::new(tonic_body_from_frames(vec![
|
|
Ok(http_body::Frame::data(bytes.clone())),
|
|
Err(tonic::Status::unavailable("response stream failed")),
|
|
]));
|
|
|
|
let (response, observation) = observe_intercepted_grpc_response(response).await;
|
|
|
|
let observation_error = observation.unwrap_err();
|
|
assert!(observation_error.contains("failed to read gRPC response"));
|
|
assert!(observation_error.contains("response stream failed"));
|
|
let mut body = response.into_body();
|
|
let data = body.frame().await.unwrap().unwrap();
|
|
assert_eq!(data.data_ref(), Some(&bytes));
|
|
let status = body.frame().await.unwrap().unwrap_err();
|
|
assert_eq!(status.code(), tonic::Code::Unavailable);
|
|
assert_eq!(status.message(), "response stream failed");
|
|
assert!(body.frame().await.is_none());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn post_commit_decode_failure_preserves_committed_response() {
|
|
let (runtime, interceptor_task) = post_commit_test_runtime().await;
|
|
let committed = Arc::new(AtomicUsize::new(0));
|
|
let committed_for_service = committed.clone();
|
|
let committed_body = Bytes::from_static(b"malformed committed gRPC response");
|
|
let committed_body_for_service = committed_body.clone();
|
|
let inner = tower::service_fn(move |_request: Request<BoxBody>| {
|
|
let committed = committed_for_service.clone();
|
|
let body = committed_body_for_service.clone();
|
|
async move {
|
|
committed.fetch_add(1, Ordering::SeqCst);
|
|
let mut trailers = http::HeaderMap::new();
|
|
trailers.insert("grpc-status", HeaderValue::from_static("0"));
|
|
Ok::<_, Infallible>(Response::new(tonic_body_from_bytes_and_trailers(
|
|
body,
|
|
Some(trailers),
|
|
)))
|
|
}
|
|
});
|
|
let mut service = GatewayInterceptorGrpcService::new(inner, Some(runtime), None);
|
|
let request_body = grpc_frame(&CreateSandboxRequest::default().encode_to_vec());
|
|
let request = Request::builder()
|
|
.uri("/openshell.v1.OpenShell/CreateSandbox")
|
|
.body(boxed_body_from_bytes(request_body))
|
|
.unwrap();
|
|
|
|
let response = service.ready().await.unwrap().call(request).await.unwrap();
|
|
|
|
assert_eq!(committed.load(Ordering::SeqCst), 1);
|
|
let collected = response.into_body().collect().await.unwrap();
|
|
assert_eq!(collected.to_bytes(), committed_body);
|
|
interceptor_task.abort();
|
|
}
|
|
|
|
#[derive(Clone, Default)]
|
|
struct ReplayTestInterceptor {
|
|
modifications: Arc<AtomicUsize>,
|
|
validations: Arc<AtomicUsize>,
|
|
observations: Arc<AtomicUsize>,
|
|
deny: Arc<std::sync::atomic::AtomicBool>,
|
|
name: Arc<Mutex<String>>,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl GatewayInterceptor for ReplayTestInterceptor {
|
|
async fn describe(
|
|
&self,
|
|
request: tonic::Request<DescribeRequest>,
|
|
) -> Result<tonic::Response<InterceptorManifest>, tonic::Status> {
|
|
let mut manifest = PostCommitTestInterceptor
|
|
.describe(request)
|
|
.await?
|
|
.into_inner();
|
|
manifest.bindings.push(InterceptorBinding {
|
|
id: "validate-create".into(),
|
|
selector: manifest.bindings[0].selector.clone(),
|
|
phases: vec![
|
|
GatewayInterceptorPhase::ModifyOperation.into(),
|
|
GatewayInterceptorPhase::Validate.into(),
|
|
],
|
|
failure_policy: "fail_closed".into(),
|
|
});
|
|
Ok(tonic::Response::new(manifest))
|
|
}
|
|
|
|
async fn evaluate(
|
|
&self,
|
|
request: tonic::Request<InterceptorEvaluation>,
|
|
) -> Result<tonic::Response<InterceptorResult>, tonic::Status> {
|
|
use openshell_core::proto::gateway_interceptor::v1::{
|
|
JsonPatch, interceptor_evaluation::Phase,
|
|
};
|
|
let mut result = InterceptorResult {
|
|
allowed: true,
|
|
..Default::default()
|
|
};
|
|
match request.into_inner().phase.unwrap() {
|
|
Phase::ModifyOperation(_) => {
|
|
self.modifications.fetch_add(1, Ordering::SeqCst);
|
|
result.patches.push(JsonPatch {
|
|
op: "replace".into(),
|
|
path: "/name".into(),
|
|
value: Some(prost_types::Value {
|
|
kind: Some(prost_types::value::Kind::StringValue(
|
|
self.name.lock().unwrap().clone(),
|
|
)),
|
|
}),
|
|
..Default::default()
|
|
});
|
|
}
|
|
Phase::Validate(_) => {
|
|
self.validations.fetch_add(1, Ordering::SeqCst);
|
|
result.allowed = !self.deny.load(Ordering::SeqCst);
|
|
result.reason = "current policy denies creation".into();
|
|
result.status_code = "PERMISSION_DENIED".into();
|
|
}
|
|
Phase::PostCommit(_) => {
|
|
self.observations.fetch_add(1, Ordering::SeqCst);
|
|
}
|
|
}
|
|
Ok(tonic::Response::new(result))
|
|
}
|
|
|
|
async fn snapshot_provider_profiles(
|
|
&self,
|
|
request: tonic::Request<ProviderProfileSnapshotRequest>,
|
|
) -> Result<tonic::Response<ProviderProfileSnapshot>, tonic::Status> {
|
|
PostCommitTestInterceptor
|
|
.snapshot_provider_profiles(request)
|
|
.await
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn mutation_replay_revalidates_interceptors_and_observes_commit_once() {
|
|
use openshell_core::proto::{
|
|
SandboxResponse, SandboxSpec, open_shell_server::OpenShellServer,
|
|
};
|
|
let interceptor = ReplayTestInterceptor::default();
|
|
*interceptor.name.lock().unwrap() = "intercepted-create".into();
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let service = interceptor.clone();
|
|
let task = tokio::spawn(async move {
|
|
tonic::transport::Server::builder()
|
|
.add_service(GatewayInterceptorServer::new(service))
|
|
.serve_with_incoming(TcpListenerStream::new(listener))
|
|
.await
|
|
.unwrap();
|
|
});
|
|
let runtime = openshell_gateway_interceptors::initialize(vec![GatewayInterceptorConfig {
|
|
name: "post-commit-test".into(),
|
|
grpc_endpoint: format!("http://{address}"),
|
|
..Default::default()
|
|
}])
|
|
.await
|
|
.unwrap();
|
|
let directory = tempfile::tempdir().unwrap();
|
|
let key = directory.path().join("key");
|
|
std::fs::write(&key, b"test-only-private-key").unwrap();
|
|
let mut state = crate::grpc::test_support::test_server_state().await;
|
|
Arc::get_mut(&mut state).unwrap().config.gateway_jwt =
|
|
Some(openshell_core::config::GatewayJwtConfig {
|
|
signing_key_path: key,
|
|
public_key_path: directory.path().join("public"),
|
|
kid_path: directory.path().join("kid"),
|
|
gateway_id: "test".into(),
|
|
ttl_secs: None,
|
|
});
|
|
let inner = OpenShellServer::new(OpenShellService::new(state.clone()));
|
|
let mut service = GatewayInterceptorGrpcService::new(inner, runtime, Some(state));
|
|
let input = CreateSandboxRequest {
|
|
name: "client-original".into(),
|
|
request_id: uuid::Uuid::new_v4().to_string(),
|
|
workspace_scope: Some(openshell_core::proto::workspace_selector(
|
|
"default".to_string(),
|
|
)),
|
|
spec: Some(SandboxSpec::default()),
|
|
..Default::default()
|
|
};
|
|
let request = || {
|
|
let principal = crate::grpc::test_support::authed_request(())
|
|
.extensions()
|
|
.get::<Principal>()
|
|
.unwrap()
|
|
.clone();
|
|
Request::builder()
|
|
.uri("/openshell.v1.OpenShell/CreateSandbox")
|
|
.header("content-type", "application/grpc")
|
|
.header("openshell-replayed", "true")
|
|
.extension(principal)
|
|
.body(boxed_body_from_bytes(grpc_frame(&input.encode_to_vec())))
|
|
.unwrap()
|
|
};
|
|
let first = service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(request())
|
|
.await
|
|
.unwrap();
|
|
assert!(!first.headers().contains_key("openshell-replayed"));
|
|
let first = first.into_body().collect().await.unwrap();
|
|
assert_eq!(first.trailers().unwrap().get("grpc-status").unwrap(), "0");
|
|
let first = decode_unary_grpc_message::<SandboxResponse>(&first.to_bytes()).unwrap();
|
|
assert_eq!(
|
|
first
|
|
.sandbox
|
|
.as_ref()
|
|
.unwrap()
|
|
.metadata
|
|
.as_ref()
|
|
.unwrap()
|
|
.name,
|
|
"intercepted-create"
|
|
);
|
|
let replay = service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(request())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(replay.headers().get("openshell-replayed").unwrap(), "true");
|
|
let replay = replay.into_body().collect().await.unwrap();
|
|
assert_eq!(
|
|
decode_unary_grpc_message::<SandboxResponse>(&replay.to_bytes()).unwrap(),
|
|
first
|
|
);
|
|
assert_eq!(interceptor.modifications.load(Ordering::SeqCst), 2);
|
|
assert_eq!(interceptor.validations.load(Ordering::SeqCst), 2);
|
|
assert_eq!(interceptor.observations.load(Ordering::SeqCst), 1);
|
|
interceptor.deny.store(true, Ordering::SeqCst);
|
|
let denied = service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(request())
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(denied.headers().get("grpc-status").unwrap(), "7");
|
|
assert_eq!(interceptor.validations.load(Ordering::SeqCst), 3);
|
|
assert_eq!(interceptor.observations.load(Ordering::SeqCst), 1);
|
|
task.abort();
|
|
}
|
|
|
|
#[test]
|
|
fn grpc_trailer_status_takes_precedence_over_headers() {
|
|
let response = Response::builder()
|
|
.header("grpc-status", "0")
|
|
.body(())
|
|
.unwrap();
|
|
let mut trailers = http::HeaderMap::new();
|
|
trailers.insert("grpc-status", HeaderValue::from_static("7"));
|
|
|
|
assert_eq!(
|
|
grpc_status_from_response_and_trailers(&response, Some(&trailers)),
|
|
"7"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn http_response_includes_request_id() {
|
|
let addr = start_http_server_with_middleware().await;
|
|
let resp = http1_get(addr, "/healthz", &[]).await;
|
|
assert_eq!(resp.status(), 200);
|
|
|
|
let request_id = resp
|
|
.headers()
|
|
.get("x-request-id")
|
|
.expect("response should include x-request-id header");
|
|
let id_str = request_id.to_str().unwrap();
|
|
uuid::Uuid::parse_str(id_str).expect("should be a valid UUID");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn http_preserves_client_request_id() {
|
|
let addr = start_http_server_with_middleware().await;
|
|
let client_id = "my-custom-correlation-id";
|
|
let resp = http1_get(addr, "/healthz", &[("x-request-id", client_id)]).await;
|
|
assert_eq!(resp.status(), 200);
|
|
|
|
let request_id = resp
|
|
.headers()
|
|
.get("x-request-id")
|
|
.expect("response should include x-request-id header");
|
|
assert_eq!(request_id.to_str().unwrap(), client_id);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn each_request_gets_unique_id() {
|
|
let addr = start_http_server_with_middleware().await;
|
|
|
|
let mut ids = Vec::new();
|
|
for _ in 0..3 {
|
|
let resp = http1_get(addr, "/healthz", &[]).await;
|
|
let id = resp
|
|
.headers()
|
|
.get("x-request-id")
|
|
.unwrap()
|
|
.to_str()
|
|
.unwrap()
|
|
.to_string();
|
|
ids.push(id);
|
|
}
|
|
|
|
assert_ne!(ids[0], ids[1]);
|
|
assert_ne!(ids[1], ids[2]);
|
|
assert_ne!(ids[0], ids[2]);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn grpc_path_includes_request_id() {
|
|
let addr = start_http_server_with_middleware().await;
|
|
let resp = http1_get(
|
|
addr,
|
|
"/openshell.v1.OpenShell/Health",
|
|
&[
|
|
("content-type", "application/grpc"),
|
|
("x-request-id", "grpc-corr-id"),
|
|
],
|
|
)
|
|
.await;
|
|
|
|
let request_id = resp
|
|
.headers()
|
|
.get("x-request-id")
|
|
.expect("gRPC-routed response should include x-request-id header");
|
|
assert_eq!(request_id.to_str().unwrap(), "grpc-corr-id");
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct CountingGrpcService {
|
|
calls: Arc<AtomicUsize>,
|
|
}
|
|
|
|
impl Service<Request<()>> for CountingGrpcService {
|
|
type Response = Response<tonic::body::Body>;
|
|
type Error = Infallible;
|
|
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
|
|
|
|
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
|
Poll::Ready(Ok(()))
|
|
}
|
|
|
|
fn call(&mut self, _req: Request<()>) -> Self::Future {
|
|
self.calls.fetch_add(1, Ordering::Relaxed);
|
|
std::future::ready(Ok(Response::new(tonic::body::Body::empty())))
|
|
}
|
|
}
|
|
|
|
/// Inner service that is never ready, used to prove the rate limiter does
|
|
/// not wait on inner-service backpressure when it is already exhausted.
|
|
/// Counts `call` invocations so tests can assert the limiter never forwards
|
|
/// to an inner service that did not report readiness.
|
|
#[derive(Clone)]
|
|
struct PendingInnerService {
|
|
calls: Arc<AtomicUsize>,
|
|
}
|
|
|
|
impl PendingInnerService {
|
|
fn new() -> Self {
|
|
Self {
|
|
calls: Arc::new(AtomicUsize::new(0)),
|
|
}
|
|
}
|
|
}
|
|
|
|
impl Service<Request<()>> for PendingInnerService {
|
|
type Response = Response<tonic::body::Body>;
|
|
type Error = Infallible;
|
|
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
|
|
|
|
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
|
Poll::Pending
|
|
}
|
|
|
|
fn call(&mut self, _req: Request<()>) -> Self::Future {
|
|
self.calls.fetch_add(1, Ordering::Relaxed);
|
|
std::future::ready(Ok(Response::new(tonic::body::Body::empty())))
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn grpc_rate_limit_poll_ready_short_circuits_exhausted_limiter() {
|
|
// An exhausted limiter must report ready even when the inner service is
|
|
// pending, so `call` returns RESOURCE_EXHAUSTED instead of waiting on
|
|
// inner backpressure.
|
|
let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60));
|
|
let limiter = GrpcRateLimiter::from_config(&config).expect("limiter should be enabled");
|
|
// Consume the single token so the limiter is exhausted.
|
|
assert!(limiter.allow());
|
|
|
|
let mut exhausted = GrpcRateLimitService::new(PendingInnerService::new(), Some(limiter));
|
|
let mut cx = Context::from_waker(std::task::Waker::noop());
|
|
assert!(
|
|
matches!(exhausted.poll_ready(&mut cx), Poll::Ready(Ok(()))),
|
|
"exhausted limiter should report ready despite a pending inner service",
|
|
);
|
|
let response = exhausted.call(Request::new(())).await.unwrap();
|
|
assert_eq!(grpc_status_from_response(&response), "8");
|
|
|
|
// A limiter with capacity must still respect inner backpressure.
|
|
let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60));
|
|
let limiter = GrpcRateLimiter::from_config(&config);
|
|
let mut with_capacity = GrpcRateLimitService::new(PendingInnerService::new(), limiter);
|
|
assert!(
|
|
with_capacity.poll_ready(&mut cx).is_pending(),
|
|
"limiter with capacity should defer to the pending inner service",
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn grpc_rate_limit_call_rejects_after_poll_ready_short_circuit_despite_window_rollover() {
|
|
// Regression: when `poll_ready` reports synthetic readiness for an
|
|
// exhausted limiter, it does NOT poll the inner service. If the
|
|
// rate-limit window then rolls over before `call`, the request must
|
|
// still be rejected rather than forwarded to an inner service that
|
|
// never reported readiness (a Tower contract violation).
|
|
let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60));
|
|
let limiter = GrpcRateLimiter::from_config(&config).expect("limiter should be enabled");
|
|
// Exhaust the single token.
|
|
assert!(limiter.allow());
|
|
|
|
// Pending inner service: its `poll_ready` never reports ready and its
|
|
// `call` increments a counter. A ready result from the wrapper
|
|
// therefore proves the limiter short-circuited rather than delegating,
|
|
// and `calls == 0` proves the wrapper never forwarded.
|
|
let inner = PendingInnerService::new();
|
|
let calls = inner.calls.clone();
|
|
let mut service = GrpcRateLimitService::new(inner, Some(limiter.clone()));
|
|
|
|
// poll_ready short-circuits the exhausted limiter and records synthetic
|
|
// readiness without polling the inner service.
|
|
let mut cx = Context::from_waker(std::task::Waker::noop());
|
|
assert!(
|
|
matches!(service.poll_ready(&mut cx), Poll::Ready(Ok(()))),
|
|
"exhausted limiter should report ready despite a pending inner service",
|
|
);
|
|
|
|
// The window rolls over between poll_ready and call: the limiter now
|
|
// has capacity again.
|
|
{
|
|
let mut state = limiter
|
|
.state
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
state.window_started = state
|
|
.window_started
|
|
.checked_sub(Duration::from_secs(61))
|
|
.expect("test window rewind should be valid");
|
|
}
|
|
|
|
let response = service.call(Request::new(())).await.unwrap();
|
|
assert_eq!(grpc_status_from_response(&response), "8");
|
|
assert_eq!(
|
|
calls.load(Ordering::Relaxed),
|
|
0,
|
|
"inner service must not be called when poll_ready short-circuited the limiter",
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn grpc_rate_limit_returns_resource_exhausted_after_limit() {
|
|
let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60));
|
|
let limiter = GrpcRateLimiter::from_config(&config);
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let mut service = GrpcRateLimitService::new(
|
|
CountingGrpcService {
|
|
calls: calls.clone(),
|
|
},
|
|
limiter,
|
|
);
|
|
|
|
let first = service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(Request::new(()))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status_from_response(&first), "0");
|
|
|
|
let second = service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(Request::new(()))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status_from_response(&second), "8");
|
|
assert_eq!(calls.load(Ordering::Relaxed), 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn grpc_rate_limit_disabled_passes_requests_through() {
|
|
let config = Config::new(None).with_grpc_rate_limit(Some(0), Some(60));
|
|
let limiter = GrpcRateLimiter::from_config(&config);
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let mut service = GrpcRateLimitService::new(
|
|
CountingGrpcService {
|
|
calls: calls.clone(),
|
|
},
|
|
limiter,
|
|
);
|
|
|
|
for _ in 0..3 {
|
|
let response = service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(Request::new(()))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status_from_response(&response), "0");
|
|
}
|
|
assert_eq!(calls.load(Ordering::Relaxed), 3);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn grpc_rate_limit_resets_after_window() {
|
|
let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60));
|
|
let limiter = GrpcRateLimiter::from_config(&config).expect("limiter should be enabled");
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let mut service = GrpcRateLimitService::new(
|
|
CountingGrpcService {
|
|
calls: calls.clone(),
|
|
},
|
|
Some(limiter.clone()),
|
|
);
|
|
|
|
let first = service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(Request::new(()))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status_from_response(&first), "0");
|
|
|
|
{
|
|
let mut state = limiter
|
|
.state
|
|
.lock()
|
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
|
state.window_started = state
|
|
.window_started
|
|
.checked_sub(Duration::from_secs(61))
|
|
.expect("test window rewind should be valid");
|
|
}
|
|
|
|
let second = service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(Request::new(()))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status_from_response(&second), "0");
|
|
assert_eq!(calls.load(Ordering::Relaxed), 2);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn grpc_rate_limit_state_is_shared_across_service_clones() {
|
|
let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60));
|
|
let limiter = GrpcRateLimiter::from_config(&config);
|
|
let calls = Arc::new(AtomicUsize::new(0));
|
|
let mut first_service = GrpcRateLimitService::new(
|
|
CountingGrpcService {
|
|
calls: calls.clone(),
|
|
},
|
|
limiter.clone(),
|
|
);
|
|
let mut second_service = GrpcRateLimitService::new(
|
|
CountingGrpcService {
|
|
calls: calls.clone(),
|
|
},
|
|
limiter,
|
|
);
|
|
|
|
let first = first_service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(Request::new(()))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status_from_response(&first), "0");
|
|
|
|
let second = second_service
|
|
.ready()
|
|
.await
|
|
.unwrap()
|
|
.call(Request::new(()))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status_from_response(&second), "8");
|
|
assert_eq!(calls.load(Ordering::Relaxed), 1);
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct TraceBuf(Arc<Mutex<Vec<u8>>>);
|
|
|
|
impl std::io::Write for TraceBuf {
|
|
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
|
|
self.0.lock().unwrap().extend_from_slice(buf);
|
|
Ok(buf.len())
|
|
}
|
|
|
|
fn flush(&mut self) -> std::io::Result<()> {
|
|
Ok(())
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn request_id_appears_in_trace_span() {
|
|
use tracing_subscriber::fmt::format::FmtSpan;
|
|
|
|
let log_buf: Arc<Mutex<Vec<u8>>> = Arc::new(Mutex::new(Vec::new()));
|
|
let writer = TraceBuf(log_buf.clone());
|
|
|
|
let fmt_layer = tracing_subscriber::fmt::layer()
|
|
.with_writer(move || writer.clone())
|
|
.with_ansi(false)
|
|
.with_span_events(FmtSpan::CLOSE);
|
|
|
|
let subscriber = {
|
|
use tracing_subscriber::layer::SubscriberExt as _;
|
|
tracing_subscriber::registry().with(fmt_layer)
|
|
};
|
|
{
|
|
let _traced = crate::otel_tracing::test_exporter::install_scoped(subscriber);
|
|
|
|
let req = Request::builder()
|
|
.uri("/test-path")
|
|
.header("x-request-id", "trace-test-id-12345")
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
let span = make_request_span(&req);
|
|
drop(span.enter());
|
|
drop(span);
|
|
}
|
|
|
|
let output = String::from_utf8(log_buf.lock().unwrap().clone()).unwrap();
|
|
assert!(
|
|
output.contains("trace-test-id-12345"),
|
|
"trace output should contain the request_id recorded in the span, got: {output}"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn polled_paths_get_debug_request_spans() {
|
|
let _traced = crate::otel_tracing::test_exporter::install_traced();
|
|
let level = |path: &str| {
|
|
let req = Request::builder()
|
|
.uri(path)
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
*make_request_span(&req)
|
|
.metadata()
|
|
.expect("span enabled")
|
|
.level()
|
|
};
|
|
|
|
for path in [
|
|
"/healthz",
|
|
"/openshell.v1.OpenShell/GetSandboxConfig",
|
|
"/openshell.v1.OpenShell/ReportProviderReadiness",
|
|
"/openshell.v1.OpenShell/PeerReportProviderReadiness",
|
|
] {
|
|
assert_eq!(level(path), tracing::Level::DEBUG, "{path}");
|
|
}
|
|
assert_eq!(
|
|
level("/openshell.v1.OpenShell/CreateSandbox"),
|
|
tracing::Level::INFO
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn polled_path_responses_log_at_debug() {
|
|
let log_buf: Arc<Mutex<Vec<u8>>> = Arc::new(Mutex::new(Vec::new()));
|
|
let writer = TraceBuf(log_buf.clone());
|
|
let subscriber = {
|
|
use tracing_subscriber::layer::SubscriberExt as _;
|
|
tracing_subscriber::registry()
|
|
.with(tracing_subscriber::filter::LevelFilter::INFO)
|
|
.with(
|
|
tracing_subscriber::fmt::layer()
|
|
.with_writer(move || writer.clone())
|
|
.with_ansi(false),
|
|
)
|
|
};
|
|
{
|
|
let _traced = crate::otel_tracing::test_exporter::install_scoped(subscriber);
|
|
let respond = |path: &str, status: u16| {
|
|
let req = Request::builder()
|
|
.uri(path)
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
let res = Response::builder()
|
|
.status(status)
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
log_response(&res, Duration::from_millis(1), &make_request_span(&req));
|
|
};
|
|
respond("/openshell.v1.OpenShell/GetSandboxConfig", 200);
|
|
respond("/healthz", 200);
|
|
respond("/openshell.v1.OpenShell/CreateSandbox", 201);
|
|
respond("/healthz", 503);
|
|
}
|
|
|
|
let output = String::from_utf8(log_buf.lock().unwrap().clone()).unwrap();
|
|
let lines: Vec<&str> = output.lines().collect();
|
|
assert_eq!(lines.len(), 2, "got: {output}");
|
|
assert!(
|
|
lines[0].contains("INFO") && lines[0].contains("status=201"),
|
|
"got: {output}"
|
|
);
|
|
assert!(
|
|
lines[1].contains("WARN") && lines[1].contains("status=503"),
|
|
"got: {output}"
|
|
);
|
|
}
|
|
|
|
/// The `TraceLayer` creates the server span, so no gRPC handler needs
|
|
/// `#[instrument]`. The request ID carries into it so a trace can be
|
|
/// correlated with the gateway's logs.
|
|
#[tokio::test]
|
|
async fn request_span_exports_over_otlp_with_request_id() {
|
|
use crate::otel_tracing::test_exporter;
|
|
|
|
let traced = test_exporter::install_traced();
|
|
let req = Request::builder()
|
|
.uri("/openshell.v1.OpenShell/CreateSandbox")
|
|
.header("x-request-id", "otlp-req-id-9876")
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
let span = make_request_span(&req);
|
|
drop(span.enter());
|
|
drop(span);
|
|
|
|
let spans = traced.finished_spans();
|
|
let span = spans
|
|
.iter()
|
|
.find(|s| s.name == "openshell.v1.OpenShell/CreateSandbox")
|
|
.unwrap_or_else(|| {
|
|
panic!(
|
|
"the per-request span is recorded under its RPC name, got {:?}",
|
|
spans.iter().map(|s| &s.name).collect::<Vec<_>>()
|
|
)
|
|
});
|
|
assert_eq!(
|
|
test_exporter::attribute(span, "request_id").as_deref(),
|
|
Some("otlp-req-id-9876"),
|
|
);
|
|
assert_eq!(
|
|
test_exporter::attribute(span, "path").as_deref(),
|
|
Some("/openshell.v1.OpenShell/CreateSandbox"),
|
|
);
|
|
assert_eq!(
|
|
span.span_kind,
|
|
opentelemetry::trace::SpanKind::Server,
|
|
"trace UIs lay this out as a served call, not an internal operation"
|
|
);
|
|
test_exporter::assert_is_root(span);
|
|
assert_eq!(
|
|
test_exporter::attribute(span, "rpc.system").as_deref(),
|
|
Some("grpc"),
|
|
);
|
|
assert_eq!(
|
|
test_exporter::attribute(span, "rpc.service").as_deref(),
|
|
Some("openshell.v1.OpenShell"),
|
|
);
|
|
assert_eq!(
|
|
test_exporter::attribute(span, "rpc.method").as_deref(),
|
|
Some("CreateSandbox"),
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn request_span_continues_the_incoming_trace() {
|
|
use crate::otel_tracing::test_exporter;
|
|
|
|
let traced = test_exporter::install_traced();
|
|
let req = Request::builder()
|
|
.uri("/openshell.v1.OpenShell/CreateSandbox")
|
|
.header(
|
|
"traceparent",
|
|
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01",
|
|
)
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
let span = make_request_span(&req);
|
|
drop(span.enter());
|
|
drop(span);
|
|
|
|
let span = traced.span_with(
|
|
"openshell.v1.OpenShell/CreateSandbox",
|
|
"rpc.method",
|
|
"CreateSandbox",
|
|
);
|
|
assert_eq!(
|
|
span.span_context.trace_id().to_string(),
|
|
"4bf92f3577b34da6a3ce929d0e0e4736"
|
|
);
|
|
assert_eq!(
|
|
span.parent_span_id.to_string(),
|
|
"00f067aa0ba902b7",
|
|
"the server span is a child of the caller's span"
|
|
);
|
|
}
|
|
|
|
/// A failed request must be distinguishable from a successful one in a
|
|
/// trace UI, which keys off span status rather than a logged field.
|
|
#[tokio::test]
|
|
async fn request_spans_record_the_response_outcome() {
|
|
use crate::otel_tracing::test_exporter;
|
|
|
|
let traced = test_exporter::install_traced();
|
|
for (path, status) in [
|
|
("/openshell.v1.OpenShell/CreateSandbox", 500),
|
|
("/openshell.v1.OpenShell/ListSandboxes", 200),
|
|
] {
|
|
let req = Request::builder()
|
|
.uri(path)
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
let span = make_request_span(&req);
|
|
let res = Response::builder()
|
|
.status(status)
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
let entered = span.enter();
|
|
log_response(&res, Duration::from_millis(3), &span);
|
|
drop(entered);
|
|
drop(span);
|
|
}
|
|
|
|
let spans = traced.finished_spans();
|
|
let failed = spans
|
|
.iter()
|
|
.find(|s| s.name == "openshell.v1.OpenShell/CreateSandbox")
|
|
.expect("failed request span recorded");
|
|
let succeeded = spans
|
|
.iter()
|
|
.find(|s| s.name == "openshell.v1.OpenShell/ListSandboxes")
|
|
.expect("successful request span recorded");
|
|
|
|
assert_eq!(
|
|
test_exporter::attribute(failed, "http.response.status_code").as_deref(),
|
|
Some("500"),
|
|
"the response status is an attribute, not only a log field"
|
|
);
|
|
assert!(
|
|
matches!(failed.status, opentelemetry::trace::Status::Error { .. }),
|
|
"the span carries error status so trace UIs flag it, got {:?}",
|
|
failed.status
|
|
);
|
|
assert!(
|
|
!matches!(succeeded.status, opentelemetry::trace::Status::Error { .. }),
|
|
"got {:?}",
|
|
succeeded.status
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn request_span_records_grpc_status_from_trailers() {
|
|
use crate::otel_tracing::test_exporter;
|
|
|
|
let traced = test_exporter::install_traced();
|
|
let req = Request::builder()
|
|
.uri("/openshell.v1.OpenShell/CreateSandbox")
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
let span = make_request_span(&req);
|
|
let mut trailers = http::HeaderMap::new();
|
|
trailers.insert("grpc-status", HeaderValue::from_static("13"));
|
|
record_response_trailers(Some(&trailers), Duration::from_millis(3), &span);
|
|
drop(span);
|
|
|
|
let span = traced.span_with(
|
|
"openshell.v1.OpenShell/CreateSandbox",
|
|
"rpc.method",
|
|
"CreateSandbox",
|
|
);
|
|
assert_eq!(
|
|
test_exporter::attribute(&span, "rpc.grpc.status_code").as_deref(),
|
|
Some("13")
|
|
);
|
|
assert!(
|
|
matches!(span.status, opentelemetry::trace::Status::Error { .. }),
|
|
"a non-OK gRPC trailer marks the span as failed"
|
|
);
|
|
}
|
|
|
|
/// Without upstream trace context, each inbound entrypoint roots a trace
|
|
/// named for its RPC or HTTP method.
|
|
#[tokio::test]
|
|
async fn each_entrypoint_gets_its_own_root_span() {
|
|
use crate::otel_tracing::test_exporter;
|
|
|
|
let paths = [
|
|
"/openshell.v1.OpenShell/CreateSandbox",
|
|
"/openshell.v1.OpenShell/ListSandboxes",
|
|
"/openshell.v1.OpenShell/DeleteSandbox",
|
|
"/metrics",
|
|
];
|
|
|
|
let traced = test_exporter::install_traced();
|
|
for path in paths {
|
|
let req = Request::builder()
|
|
.uri(path)
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap();
|
|
let span = make_request_span(&req);
|
|
drop(span.enter());
|
|
drop(span);
|
|
}
|
|
|
|
let names: std::collections::BTreeSet<String> = traced
|
|
.finished_spans()
|
|
.iter()
|
|
.map(|s| s.name.to_string())
|
|
.collect();
|
|
|
|
let expected = [
|
|
"GET",
|
|
"openshell.v1.OpenShell/CreateSandbox",
|
|
"openshell.v1.OpenShell/DeleteSandbox",
|
|
"openshell.v1.OpenShell/ListSandboxes",
|
|
]
|
|
.into_iter()
|
|
.map(String::from)
|
|
.collect::<std::collections::BTreeSet<_>>();
|
|
|
|
assert!(
|
|
expected.is_subset(&names),
|
|
"each entrypoint exports under its own name, got {names:?}"
|
|
);
|
|
assert!(
|
|
!names.contains("request"),
|
|
"no entrypoint falls back to the generic callsite name, got {names:?}"
|
|
);
|
|
}
|
|
|
|
/// gRPC spans are named for the RPC, per the OpenTelemetry RPC semantic
|
|
/// conventions (`$service/$method`).
|
|
#[test]
|
|
fn grpc_request_spans_are_named_for_the_rpc() {
|
|
assert_eq!(
|
|
otel_span_name(&http::Method::POST, "/openshell.v1.OpenShell/CreateSandbox"),
|
|
"openshell.v1.OpenShell/CreateSandbox"
|
|
);
|
|
}
|
|
|
|
/// Non-RPC paths use a low-cardinality method-only name because sandbox
|
|
/// application routes are opaque to the gateway.
|
|
#[test]
|
|
fn http_request_spans_do_not_include_the_literal_path() {
|
|
assert_eq!(otel_span_name(&http::Method::GET, "/users/12345"), "GET");
|
|
assert_eq!(otel_span_name(&http::Method::GET, "/users/67890"), "GET");
|
|
}
|
|
|
|
/// A path with no service segment must not produce a span named after a
|
|
/// stray slash or an empty string.
|
|
#[test]
|
|
fn bare_paths_fall_back_to_the_http_shape() {
|
|
assert_eq!(otel_span_name(&http::Method::GET, "/"), "GET");
|
|
assert_eq!(otel_span_name(&http::Method::POST, "/Foo"), "POST");
|
|
}
|
|
|
|
#[test]
|
|
fn grpc_method_extracts_last_segment() {
|
|
assert_eq!(
|
|
grpc_method_from_path("/openshell.v1.OpenShell/CreateSandbox"),
|
|
"CreateSandbox"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn grpc_method_handles_bare_path() {
|
|
assert_eq!(grpc_method_from_path("Health"), "Health");
|
|
}
|
|
|
|
#[test]
|
|
fn grpc_method_handles_single_slash() {
|
|
assert_eq!(grpc_method_from_path("/"), "");
|
|
}
|
|
|
|
#[test]
|
|
fn grpc_method_handles_empty_string() {
|
|
assert_eq!(grpc_method_from_path(""), "");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn grpc_router_dispatches_gateway_and_reflection_paths() {
|
|
#[derive(Clone)]
|
|
struct RouteRecorder {
|
|
name: &'static str,
|
|
calls: Arc<Mutex<Vec<&'static str>>>,
|
|
}
|
|
|
|
impl<B: Send + 'static> Service<Request<B>> for RouteRecorder {
|
|
type Response = Response<tonic::body::Body>;
|
|
type Error = Infallible;
|
|
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
|
|
|
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
|
Poll::Ready(Ok(()))
|
|
}
|
|
|
|
fn call(&mut self, _req: Request<B>) -> Self::Future {
|
|
self.calls.lock().unwrap().push(self.name);
|
|
Box::pin(async { Ok(Response::new(tonic::body::Body::empty())) })
|
|
}
|
|
}
|
|
|
|
let calls = Arc::new(Mutex::new(Vec::new()));
|
|
let service = |name| RouteRecorder {
|
|
name,
|
|
calls: calls.clone(),
|
|
};
|
|
let mut router = GrpcRouter::new(service("openshell"), service("reflection"));
|
|
|
|
for path in [
|
|
"/openshell.v1.OpenShell/Health",
|
|
"/grpc.reflection.v1.ServerReflection/ServerReflectionInfo",
|
|
] {
|
|
router
|
|
.call(
|
|
Request::builder()
|
|
.uri(path)
|
|
.body(Empty::<Bytes>::new())
|
|
.unwrap(),
|
|
)
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
assert_eq!(*calls.lock().unwrap(), vec!["openshell", "reflection"]);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reflection_protocol_serves_complete_public_descriptors_and_recovers_from_errors() {
|
|
use crate::auth::authenticator::test_support::MockAuthenticator;
|
|
use prost_reflect::{DescriptorPool, Value};
|
|
use tonic_reflection::pb::v1::{
|
|
ServerReflectionRequest, server_reflection_client::ServerReflectionClient,
|
|
server_reflection_request::MessageRequest, server_reflection_response::MessageResponse,
|
|
};
|
|
|
|
let reflection =
|
|
crate::reflection::build_gateway_reflection_service(&Config::new(None)).unwrap();
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let addr = listener.local_addr().unwrap();
|
|
let unrouted = tower::service_fn(|_request: Request<BoxBody>| async {
|
|
Ok::<_, Infallible>(tonic::Status::unimplemented("test fallback").into_http())
|
|
});
|
|
let rejecting_oidc = Arc::new(MockAuthenticator::returning(Err(
|
|
crate::auth::authenticator::AuthRejection::new(
|
|
AuthMechanism::Oidc,
|
|
AuthFailureReason::InvalidCredential,
|
|
tonic::Status::unauthenticated("OIDC credentials required"),
|
|
),
|
|
)));
|
|
let grpc = AuthGrpcRouter::with_peer_identity(
|
|
GrpcRouter::new(unrouted, reflection),
|
|
Some(AuthenticatorChain::new(vec![rejecting_oidc])),
|
|
None,
|
|
None,
|
|
true,
|
|
false,
|
|
);
|
|
let service = MultiplexedService::new(grpc, unrouted);
|
|
let server = tokio::spawn(async move {
|
|
loop {
|
|
let (stream, _) = listener.accept().await.unwrap();
|
|
let service = service.clone();
|
|
tokio::spawn(async move {
|
|
Builder::new(TokioExecutor::new())
|
|
.serve_connection(TokioIo::new(stream), service)
|
|
.await
|
|
.unwrap();
|
|
});
|
|
}
|
|
});
|
|
|
|
let channel = tonic::transport::Channel::from_shared(format!("http://{addr}"))
|
|
.unwrap()
|
|
.connect()
|
|
.await
|
|
.unwrap();
|
|
let mut client = ServerReflectionClient::new(channel);
|
|
let list_request = ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::ListServices(String::new())),
|
|
};
|
|
let descriptor_request = ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::FileContainingSymbol(
|
|
"openshell.v1.OpenShell".to_string(),
|
|
)),
|
|
};
|
|
let missing_request = ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::FileContainingSymbol(
|
|
"openshell.v1.DoesNotExist".to_string(),
|
|
)),
|
|
};
|
|
let list_after_error_request = ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::ListServices(String::new())),
|
|
};
|
|
let mut responses = client
|
|
.server_reflection_info(tokio_stream::iter([
|
|
list_request,
|
|
descriptor_request,
|
|
missing_request,
|
|
list_after_error_request,
|
|
]))
|
|
.await
|
|
.unwrap()
|
|
.into_inner();
|
|
let response = responses.message().await.unwrap().unwrap();
|
|
let Some(MessageResponse::ListServicesResponse(response)) = response.message_response
|
|
else {
|
|
panic!("expected a reflection list-services response");
|
|
};
|
|
let mut names: Vec<_> = response
|
|
.service
|
|
.into_iter()
|
|
.map(|service| service.name)
|
|
.collect();
|
|
names.sort();
|
|
|
|
assert_eq!(names, vec!["openshell.v1.OpenShell"]);
|
|
|
|
let descriptor_response = responses.message().await.unwrap().unwrap();
|
|
let Some(MessageResponse::FileDescriptorResponse(response)) =
|
|
descriptor_response.message_response
|
|
else {
|
|
panic!("expected a reflection file-descriptor response");
|
|
};
|
|
let mut reflected_pool = DescriptorPool::new();
|
|
for descriptor in &response.file_descriptor_proto {
|
|
reflected_pool
|
|
.decode_file_descriptor_proto(descriptor.as_slice())
|
|
.unwrap();
|
|
}
|
|
assert!(
|
|
reflected_pool
|
|
.get_service_by_name("openshell.v1.OpenShell")
|
|
.is_some(),
|
|
"a fresh descriptor pool must resolve the advertised service"
|
|
);
|
|
assert!(
|
|
reflected_pool
|
|
.get_message_by_name("openshell.v1.HealthRequest")
|
|
.is_some(),
|
|
"the response must include imported public message descriptors"
|
|
);
|
|
|
|
let source_pool = DescriptorPool::decode(openshell_core::FILE_DESCRIPTOR_SET).unwrap();
|
|
let source_auth = source_pool
|
|
.get_extension_by_name("openshell.options.v1.authorization")
|
|
.unwrap();
|
|
let reflected_auth = reflected_pool
|
|
.get_extension_by_name("openshell.options.v1.authorization")
|
|
.unwrap();
|
|
let source_health = source_pool
|
|
.get_service_by_name("openshell.v1.OpenShell")
|
|
.unwrap()
|
|
.methods()
|
|
.find(|method| method.name() == "Health")
|
|
.unwrap();
|
|
let reflected_health = reflected_pool
|
|
.get_service_by_name("openshell.v1.OpenShell")
|
|
.unwrap()
|
|
.methods()
|
|
.find(|method| method.name() == "Health")
|
|
.unwrap();
|
|
let source_options = source_health.options();
|
|
let reflected_options = reflected_health.options();
|
|
let Value::Message(source_auth_value) = &*source_options.get_extension(&source_auth) else {
|
|
panic!("source authorization option must be a message");
|
|
};
|
|
let Value::Message(reflected_auth_value) =
|
|
&*reflected_options.get_extension(&reflected_auth)
|
|
else {
|
|
panic!("reflected authorization option must be a message");
|
|
};
|
|
assert_eq!(
|
|
source_auth_value.encode_to_vec(),
|
|
reflected_auth_value.encode_to_vec()
|
|
);
|
|
|
|
let source_secret = source_pool
|
|
.get_extension_by_name("openshell.options.v1.secret")
|
|
.unwrap();
|
|
let reflected_secret = reflected_pool
|
|
.get_extension_by_name("openshell.options.v1.secret")
|
|
.unwrap();
|
|
let source_field = source_pool
|
|
.get_message_by_name("openshell.v1.TcpForwardInit")
|
|
.unwrap()
|
|
.get_field_by_name("authorization_token")
|
|
.unwrap();
|
|
let reflected_field = reflected_pool
|
|
.get_message_by_name("openshell.v1.TcpForwardInit")
|
|
.unwrap()
|
|
.get_field_by_name("authorization_token")
|
|
.unwrap();
|
|
assert_eq!(
|
|
source_field.options().get_extension(&source_secret),
|
|
reflected_field.options().get_extension(&reflected_secret),
|
|
"secret-field annotations must retain their original value"
|
|
);
|
|
|
|
let error_response = responses.message().await.unwrap().unwrap();
|
|
let Some(MessageResponse::ErrorResponse(error)) = error_response.message_response else {
|
|
panic!("expected an in-band reflection error response");
|
|
};
|
|
assert_eq!(error.error_code, tonic::Code::NotFound as i32);
|
|
|
|
let response_after_error = responses.message().await.unwrap().unwrap();
|
|
assert!(matches!(
|
|
response_after_error.message_response,
|
|
Some(MessageResponse::ListServicesResponse(_))
|
|
));
|
|
server.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reflection_protocol_resolves_canonical_enum_values_and_extensions() {
|
|
use prost::Message;
|
|
use tonic_reflection::pb::v1::{
|
|
ExtensionRequest, ServerReflectionRequest,
|
|
server_reflection_client::ServerReflectionClient,
|
|
server_reflection_request::MessageRequest, server_reflection_response::MessageResponse,
|
|
};
|
|
|
|
let reflection =
|
|
crate::reflection::build_gateway_reflection_service(&Config::new(None)).unwrap();
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let addr = listener.local_addr().unwrap();
|
|
let unrouted = tower::service_fn(|_request: Request<BoxBody>| async {
|
|
Ok::<_, Infallible>(tonic::Status::unimplemented("test fallback").into_http())
|
|
});
|
|
let service = MultiplexedService::new(GrpcRouter::new(unrouted, reflection), unrouted);
|
|
let server = tokio::spawn(async move {
|
|
loop {
|
|
let (stream, _) = listener.accept().await.unwrap();
|
|
let service = service.clone();
|
|
tokio::spawn(async move {
|
|
Builder::new(TokioExecutor::new())
|
|
.serve_connection(TokioIo::new(stream), service)
|
|
.await
|
|
.unwrap();
|
|
});
|
|
}
|
|
});
|
|
|
|
let channel = tonic::transport::Channel::from_shared(format!("http://{addr}"))
|
|
.unwrap()
|
|
.connect()
|
|
.await
|
|
.unwrap();
|
|
let mut client = ServerReflectionClient::new(channel);
|
|
|
|
for (symbol, expected_file) in [
|
|
("openshell.v1.SANDBOX_PHASE_READY", "openshell.proto"),
|
|
(
|
|
"google.protobuf.FieldDescriptorProto.TYPE_DOUBLE",
|
|
"google/protobuf/descriptor.proto",
|
|
),
|
|
] {
|
|
let request = ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::FileContainingSymbol(symbol.to_string())),
|
|
};
|
|
let mut responses = client
|
|
.server_reflection_info(tokio_stream::iter([request]))
|
|
.await
|
|
.unwrap()
|
|
.into_inner();
|
|
let response = responses.message().await.unwrap().unwrap();
|
|
let Some(MessageResponse::FileDescriptorResponse(response)) = response.message_response
|
|
else {
|
|
panic!("expected a descriptor response for canonical enum value {symbol}");
|
|
};
|
|
let names: std::collections::BTreeSet<_> = response
|
|
.file_descriptor_proto
|
|
.iter()
|
|
.map(|descriptor| {
|
|
prost_types::FileDescriptorProto::decode(descriptor.as_slice())
|
|
.unwrap()
|
|
.name
|
|
.unwrap()
|
|
})
|
|
.collect();
|
|
assert!(names.contains(expected_file));
|
|
}
|
|
|
|
for (containing_type, extension_number) in [
|
|
("google.protobuf.MethodOptions", 50_000),
|
|
("google.protobuf.FieldOptions", 50_001),
|
|
] {
|
|
let request = ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::FileContainingExtension(ExtensionRequest {
|
|
containing_type: containing_type.to_string(),
|
|
extension_number,
|
|
})),
|
|
};
|
|
let mut responses = client
|
|
.server_reflection_info(tokio_stream::iter([request]))
|
|
.await
|
|
.unwrap()
|
|
.into_inner();
|
|
let response = responses.message().await.unwrap().unwrap();
|
|
let Some(MessageResponse::FileDescriptorResponse(response)) = response.message_response
|
|
else {
|
|
panic!("expected a descriptor response for extension {extension_number}");
|
|
};
|
|
let names: std::collections::BTreeSet<_> = response
|
|
.file_descriptor_proto
|
|
.iter()
|
|
.map(|descriptor| {
|
|
prost_types::FileDescriptorProto::decode(descriptor.as_slice())
|
|
.unwrap()
|
|
.name
|
|
.unwrap()
|
|
})
|
|
.collect();
|
|
assert!(names.contains("options.proto"));
|
|
}
|
|
|
|
let requests = [
|
|
ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::AllExtensionNumbersOfType(
|
|
"google.protobuf.MethodOptions".to_string(),
|
|
)),
|
|
},
|
|
ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::FileContainingExtension(ExtensionRequest {
|
|
containing_type: "google.protobuf.MethodOptions".to_string(),
|
|
extension_number: 49_999,
|
|
})),
|
|
},
|
|
ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::ListServices(String::new())),
|
|
},
|
|
];
|
|
let mut responses = client
|
|
.server_reflection_info(tokio_stream::iter(requests))
|
|
.await
|
|
.unwrap()
|
|
.into_inner();
|
|
let extension_numbers = responses.message().await.unwrap().unwrap();
|
|
let Some(MessageResponse::AllExtensionNumbersResponse(extension_numbers)) =
|
|
extension_numbers.message_response
|
|
else {
|
|
panic!("expected an extension-number response");
|
|
};
|
|
assert_eq!(
|
|
extension_numbers.base_type_name,
|
|
"google.protobuf.MethodOptions"
|
|
);
|
|
assert_eq!(extension_numbers.extension_number, [50_000]);
|
|
|
|
let unknown_extension = responses.message().await.unwrap().unwrap();
|
|
let Some(MessageResponse::ErrorResponse(error)) = unknown_extension.message_response else {
|
|
panic!("expected an in-band error for an unknown extension");
|
|
};
|
|
assert_eq!(error.error_code, tonic::Code::NotFound as i32);
|
|
assert!(matches!(
|
|
responses.message().await.unwrap().unwrap().message_response,
|
|
Some(MessageResponse::ListServicesResponse(_))
|
|
));
|
|
|
|
server.abort();
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reflection_rate_limit_charges_each_query_on_one_stream() {
|
|
use tonic::Code;
|
|
use tonic_reflection::pb::v1::{
|
|
ServerReflectionRequest, server_reflection_client::ServerReflectionClient,
|
|
server_reflection_request::MessageRequest,
|
|
};
|
|
|
|
let config = Config::new(None).with_grpc_rate_limit(Some(1), Some(60));
|
|
let reflection = crate::reflection::build_gateway_reflection_service(&config).unwrap();
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let addr = listener.local_addr().unwrap();
|
|
let unrouted = tower::service_fn(|_request: Request<BoxBody>| async {
|
|
Ok::<_, Infallible>(tonic::Status::unimplemented("test fallback").into_http())
|
|
});
|
|
let service = MultiplexedService::new(GrpcRouter::new(unrouted, reflection), unrouted);
|
|
let server = tokio::spawn(async move {
|
|
loop {
|
|
let (stream, _) = listener.accept().await.unwrap();
|
|
let service = service.clone();
|
|
tokio::spawn(async move {
|
|
Builder::new(TokioExecutor::new())
|
|
.serve_connection(TokioIo::new(stream), service)
|
|
.await
|
|
.unwrap();
|
|
});
|
|
}
|
|
});
|
|
|
|
let channel = tonic::transport::Channel::from_shared(format!("http://{addr}"))
|
|
.unwrap()
|
|
.connect()
|
|
.await
|
|
.unwrap();
|
|
let mut client = ServerReflectionClient::new(channel);
|
|
let query = ServerReflectionRequest {
|
|
host: String::new(),
|
|
message_request: Some(MessageRequest::ListServices(String::new())),
|
|
};
|
|
let mut responses = client
|
|
.server_reflection_info(tokio_stream::iter([query.clone(), query]))
|
|
.await
|
|
.unwrap()
|
|
.into_inner();
|
|
|
|
assert!(responses.message().await.unwrap().is_some());
|
|
let status = responses
|
|
.message()
|
|
.await
|
|
.expect_err("second query on the same stream must be rate limited");
|
|
assert_eq!(status.code(), Code::ResourceExhausted);
|
|
server.abort();
|
|
}
|
|
|
|
#[test]
|
|
fn reflection_descriptor_excludes_internal_service_protos() {
|
|
let descriptors = crate::reflection::gateway_reflection_descriptors().unwrap();
|
|
let names: std::collections::BTreeSet<_> = descriptors
|
|
.iter()
|
|
.map(prost_reflect::FileDescriptor::name)
|
|
.collect();
|
|
|
|
assert!(names.contains("openshell.proto"));
|
|
assert!(names.contains("sandbox.proto"));
|
|
assert!(!names.contains("compute_driver.proto"));
|
|
assert!(!names.contains("credential_driver.proto"));
|
|
assert!(!names.contains("gateway_interceptor.proto"));
|
|
assert!(!names.contains("supervisor_middleware.proto"));
|
|
}
|
|
|
|
#[test]
|
|
fn normalize_ws_tunnel() {
|
|
assert_eq!(normalize_http_path("/_ws_tunnel"), "/_ws_tunnel");
|
|
}
|
|
|
|
#[test]
|
|
fn normalize_ws_tunnel_with_trailing() {
|
|
assert_eq!(normalize_http_path("/_ws_tunnel/foo"), "/_ws_tunnel");
|
|
}
|
|
|
|
#[test]
|
|
fn normalize_auth_path() {
|
|
assert_eq!(normalize_http_path("/auth/connect"), "/auth");
|
|
}
|
|
|
|
#[test]
|
|
fn normalize_auth_with_query() {
|
|
assert_eq!(
|
|
normalize_http_path("/auth/connect?callback_port=12345&code=AB7-X9KM"),
|
|
"/auth"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn normalize_unknown_path_collapses_to_unknown() {
|
|
assert_eq!(normalize_http_path("/random/scanner/probe"), "unknown");
|
|
}
|
|
|
|
#[test]
|
|
fn normalize_empty_path() {
|
|
assert_eq!(normalize_http_path(""), "unknown");
|
|
}
|
|
|
|
#[test]
|
|
fn normalize_root_path() {
|
|
assert_eq!(normalize_http_path("/"), "unknown");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn served_connection_reports_its_peer_and_generated_request_id() {
|
|
use openshell_core::proto::{ListSandboxesRequest, open_shell_client::OpenShellClient};
|
|
use openshell_ocsf::test_support::OcsfCapture;
|
|
|
|
let mut state = crate::grpc::test_support::test_server_state().await;
|
|
Arc::get_mut(&mut state).unwrap().config.mtls_auth.enabled = true;
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
|
let address = listener.local_addr().unwrap();
|
|
let service = MultiplexService::new(state);
|
|
tokio::spawn(async move {
|
|
let (stream, peer) = listener.accept().await.unwrap();
|
|
openshell_core::net::set_tcp_nodelay_best_effort(&stream);
|
|
let _ = service.serve(stream, ConnectionSource::Peer(peer)).await;
|
|
});
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
|
|
let channel = tonic::transport::Endpoint::from_shared(format!("http://{address}"))
|
|
.unwrap()
|
|
.connect()
|
|
.await
|
|
.unwrap();
|
|
let error = OpenShellClient::new(channel)
|
|
.list_sandboxes(ListSandboxesRequest::default())
|
|
.await
|
|
.expect_err("a request without a client certificate must be rejected");
|
|
let returned_request_id = error
|
|
.metadata()
|
|
.get("x-request-id")
|
|
.and_then(|value| value.to_str().ok())
|
|
.expect("the gateway returns the request ID it generated");
|
|
|
|
assert_eq!(error.code(), tonic::Code::Unauthenticated);
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_eq!(events[0]["src_endpoint"]["ip"], "127.0.0.1");
|
|
assert!(events[0]["src_endpoint"]["port"].as_u64().is_some());
|
|
assert_eq!(
|
|
events[0]["metadata"]["correlation_uid"],
|
|
returned_request_id
|
|
);
|
|
}
|
|
|
|
mod auth_router {
|
|
use super::*;
|
|
use crate::auth::audit::test_support::{
|
|
PEER_IP, REQUEST_ID, assert_authentication_failure, assert_boundary_finding,
|
|
assert_denial, assert_logon, assert_redacted, peer,
|
|
};
|
|
use crate::auth::authenticator::AuthRejection;
|
|
use crate::auth::authenticator::test_support::MockAuthenticator;
|
|
use crate::auth::identity::{Identity, IdentityProvider};
|
|
use crate::auth::principal::test_support::{sandbox, user};
|
|
use crate::auth::principal::{
|
|
Principal, SandboxIdentitySource, SandboxPrincipal, UserPrincipal,
|
|
};
|
|
use http_body_util::Full;
|
|
use jsonwebtoken::{Algorithm, EncodingKey, Header, encode};
|
|
use openshell_ocsf::test_support::OcsfCapture;
|
|
use serde::Serialize;
|
|
use std::sync::Arc;
|
|
use std::sync::Mutex;
|
|
use tower::Service;
|
|
|
|
type RecordedPrincipal = Arc<Mutex<Option<Principal>>>;
|
|
|
|
/// Service that snapshots the `Principal` from request extensions
|
|
/// and returns 200 OK. Used by router-level tests to assert the
|
|
/// chain's effect on the downstream service.
|
|
#[derive(Clone)]
|
|
struct PrincipalRecorder {
|
|
recorded: RecordedPrincipal,
|
|
}
|
|
|
|
impl PrincipalRecorder {
|
|
fn new() -> (Self, RecordedPrincipal) {
|
|
let recorded = Arc::new(Mutex::new(None));
|
|
(
|
|
Self {
|
|
recorded: recorded.clone(),
|
|
},
|
|
recorded,
|
|
)
|
|
}
|
|
}
|
|
|
|
impl<B: Send + 'static> Service<Request<B>> for PrincipalRecorder {
|
|
type Response = Response<tonic::body::Body>;
|
|
type Error = Infallible;
|
|
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
|
|
|
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
|
Poll::Ready(Ok(()))
|
|
}
|
|
|
|
fn call(&mut self, req: Request<B>) -> Self::Future {
|
|
let principal = req.extensions().get::<Principal>().cloned();
|
|
*self.recorded.lock().unwrap() = principal;
|
|
Box::pin(async move { Ok(Response::new(tonic::body::Body::empty())) })
|
|
}
|
|
}
|
|
|
|
fn empty_request(path: &str) -> Request<Full<Bytes>> {
|
|
Request::builder()
|
|
.uri(path)
|
|
.body(Full::new(Bytes::new()))
|
|
.unwrap()
|
|
}
|
|
|
|
fn grpc_status<B>(res: &Response<B>) -> Option<String> {
|
|
res.headers()
|
|
.get("grpc-status")
|
|
.map(|v| v.to_str().unwrap().to_string())
|
|
}
|
|
|
|
fn provider_writer_principal() -> Principal {
|
|
Principal::User(UserPrincipal {
|
|
identity: Identity {
|
|
subject: "provider-rotation-job".to_string(),
|
|
display_name: None,
|
|
roles: vec!["openshell-admin".to_string()],
|
|
scopes: vec!["provider:write".to_string()],
|
|
provider: IdentityProvider::Oidc,
|
|
},
|
|
})
|
|
}
|
|
|
|
fn mtls_identity(subject: &str) -> Identity {
|
|
Identity {
|
|
subject: subject.to_string(),
|
|
display_name: Some(subject.to_string()),
|
|
roles: vec!["openshell-user".to_string()],
|
|
scopes: vec![],
|
|
provider: IdentityProvider::Mtls,
|
|
}
|
|
}
|
|
|
|
fn operator_principal() -> OperatorPrincipal {
|
|
let mut identity = mtls_identity("operator");
|
|
identity.roles = vec!["operator".to_string()];
|
|
OperatorPrincipal {
|
|
identity,
|
|
certificate_sha256: "verified-leaf".to_string(),
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn operator_is_admin_but_not_supervisor_peer_or_unknown() {
|
|
for (method, permitted) in [
|
|
("GetProviderCredentials", true),
|
|
("CreateWorkspace", true),
|
|
("UpdateProvider", true),
|
|
("GetSandboxConfig", true),
|
|
("ConnectSupervisor", false),
|
|
("PeerRelay", false),
|
|
("IssueSandboxToken", false),
|
|
("Unknown", false),
|
|
] {
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::with_peer_identity(
|
|
recorder,
|
|
None,
|
|
Some(AuthzPolicy {
|
|
admin_role: "oidc-admin".into(),
|
|
user_role: "oidc-user".into(),
|
|
scopes_enabled: true,
|
|
}),
|
|
None,
|
|
false,
|
|
false,
|
|
)
|
|
.with_operator(Some(operator_principal()));
|
|
let response = router
|
|
.call(empty_request(&format!("/openshell.v1.OpenShell/{method}")))
|
|
.await
|
|
.unwrap();
|
|
if permitted {
|
|
assert!(grpc_status(&response).is_none());
|
|
assert!(matches!(
|
|
*seen.lock().unwrap(),
|
|
Some(Principal::Operator(_))
|
|
));
|
|
} else {
|
|
assert_eq!(grpc_status(&response).as_deref(), Some("7"));
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn user_bearer_claims_and_local_dev_cannot_export_credentials() {
|
|
let mut user = mtls_identity("ordinary-user");
|
|
user.provider = IdentityProvider::Oidc;
|
|
user.roles = vec!["operator".into(), "openshell-admin".into()];
|
|
user.scopes = vec!["openshell:all".into()];
|
|
for principal in [
|
|
Principal::User(UserPrincipal { identity: user }),
|
|
sandbox("sandbox-a"),
|
|
Principal::Peer(crate::auth::principal::PeerPrincipal {
|
|
replica_id: "replica".into(),
|
|
pod_uid: "pod".into(),
|
|
}),
|
|
] {
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(
|
|
Ok(Authentication::Accepted(principal)),
|
|
))]);
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let response = router
|
|
.call(empty_request(
|
|
"/openshell.v1.OpenShell/GetProviderCredentials",
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status(&response).as_deref(), Some("7"));
|
|
}
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, None, None);
|
|
let response = router
|
|
.call(empty_request(
|
|
"/openshell.v1.OpenShell/GetProviderCredentials",
|
|
))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status(&response).as_deref(), Some("7"));
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct LegacySandboxClaims {
|
|
sub: String,
|
|
iss: String,
|
|
aud: String,
|
|
iat: i64,
|
|
exp: i64,
|
|
sandbox_id: String,
|
|
}
|
|
|
|
fn legacy_sandbox_token(material: &openshell_bootstrap::jwt::JwtKeyMaterial) -> String {
|
|
let claims = LegacySandboxClaims {
|
|
sub: "spiffe://openshell/sandbox/sandbox-a".to_string(),
|
|
iss: "openshell-gateway:test".to_string(),
|
|
aud: "openshell-gateway:test".to_string(),
|
|
iat: 1,
|
|
exp: 0,
|
|
sandbox_id: "sandbox-a".to_string(),
|
|
};
|
|
let mut header = Header::new(Algorithm::EdDSA);
|
|
header.kid = Some(material.kid.clone());
|
|
encode(
|
|
&header,
|
|
&claims,
|
|
&EncodingKey::from_ed_pem(material.signing_key_pem.as_bytes())
|
|
.expect("signing key"),
|
|
)
|
|
.expect("legacy token")
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn mtls_peer_identity_fills_missing_principal_when_enabled() {
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::NotApplicable,
|
|
)));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::with_peer_identity(
|
|
recorder,
|
|
Some(chain),
|
|
None,
|
|
Some(mtls_identity("openshell-client")),
|
|
true,
|
|
false,
|
|
);
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(res.status(), 200);
|
|
let principal = seen.lock().unwrap().clone().expect("principal");
|
|
match principal {
|
|
Principal::User(u) => {
|
|
assert_eq!(u.identity.subject, "openshell-client");
|
|
assert_eq!(u.identity.provider, IdentityProvider::Mtls);
|
|
}
|
|
other => panic!("expected mTLS user principal, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn mtls_peer_identity_authenticates_without_chain_when_enabled() {
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::with_peer_identity(
|
|
recorder,
|
|
None,
|
|
None,
|
|
Some(mtls_identity("openshell-client")),
|
|
true,
|
|
false,
|
|
);
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(res.status(), 200);
|
|
assert!(matches!(
|
|
seen.lock().unwrap().as_ref(),
|
|
Some(Principal::User(_))
|
|
));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn mtls_auth_enabled_requires_peer_identity() {
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router =
|
|
AuthGrpcRouter::with_peer_identity(recorder, None, None, None, true, false);
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_none());
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("16"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn reflection_bypasses_oidc_and_mtls_user_authentication() {
|
|
let oidc = Arc::new(MockAuthenticator::returning(Err(AuthRejection::new(
|
|
AuthMechanism::Oidc,
|
|
AuthFailureReason::InvalidCredential,
|
|
tonic::Status::unauthenticated("OIDC credentials required"),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![oidc]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router =
|
|
AuthGrpcRouter::with_peer_identity(recorder, Some(chain), None, None, true, false);
|
|
|
|
let res = router
|
|
.call(empty_request(
|
|
"/grpc.reflection.v1.ServerReflection/ServerReflectionInfo",
|
|
))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(res.status(), 200);
|
|
assert_eq!(grpc_status(&res), None);
|
|
assert!(
|
|
seen.lock().unwrap().is_none(),
|
|
"reflection must not receive an authenticated user principal"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unauthenticated_dev_user_fills_missing_principal_when_enabled() {
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::NotApplicable,
|
|
)));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router =
|
|
AuthGrpcRouter::with_peer_identity(recorder, Some(chain), None, None, false, true);
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(res.status(), 200);
|
|
let principal = seen.lock().unwrap().clone().expect("principal");
|
|
match principal {
|
|
Principal::User(u) => {
|
|
assert_eq!(u.identity.subject, "unauthenticated-local-dev");
|
|
assert_eq!(u.identity.provider, IdentityProvider::LocalDev);
|
|
}
|
|
other => panic!("expected dev user principal, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unauthenticated_dev_user_authenticates_without_chain_when_enabled() {
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router =
|
|
AuthGrpcRouter::with_peer_identity(recorder, None, None, None, false, true);
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(res.status(), 200);
|
|
assert!(matches!(
|
|
seen.lock().unwrap().as_ref(),
|
|
Some(Principal::User(user))
|
|
if user.identity.subject == "unauthenticated-local-dev"
|
|
));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn user_principal_lands_in_request_extensions() {
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(Authentication::Accepted(
|
|
user("alice"),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let _ = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
let principal = seen.lock().unwrap().clone().expect("principal");
|
|
match principal {
|
|
Principal::User(u) => assert_eq!(u.identity.subject, "alice"),
|
|
_ => panic!("expected user principal"),
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn sandbox_principal_lands_in_request_extensions() {
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(Authentication::Accepted(
|
|
sandbox("sandbox-a"),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let _ = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ReportPolicyStatus"))
|
|
.await
|
|
.unwrap();
|
|
let captured = seen.lock().unwrap().clone();
|
|
match captured {
|
|
Some(Principal::Sandbox(p)) => assert_eq!(p.sandbox_id, "sandbox-a"),
|
|
other => panic!("expected sandbox principal, got {other:?}"),
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn sandbox_principal_can_call_allowlisted_method() {
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(Authentication::Accepted(
|
|
sandbox("sandbox-a"),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/GetSandboxConfig"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(res.status(), 200);
|
|
assert!(matches!(
|
|
seen.lock().unwrap().as_ref(),
|
|
Some(Principal::Sandbox(_))
|
|
));
|
|
}
|
|
|
|
/// A user principal — even one carrying `openshell:all` and the
|
|
/// admin role — must not reach a `sandbox`-annotated method. The
|
|
/// router enforces this from the per-handler auth-mode declarations
|
|
/// independent of RBAC.
|
|
#[tokio::test]
|
|
async fn user_principal_is_denied_on_sandbox_only_methods() {
|
|
fn admin_user() -> Principal {
|
|
Principal::User(UserPrincipal {
|
|
identity: Identity {
|
|
subject: "admin".to_string(),
|
|
display_name: None,
|
|
roles: vec!["openshell-admin".to_string()],
|
|
scopes: vec!["openshell:all".to_string()],
|
|
provider: IdentityProvider::Oidc,
|
|
},
|
|
})
|
|
}
|
|
|
|
let policy = AuthzPolicy {
|
|
admin_role: "openshell-admin".to_string(),
|
|
user_role: "openshell-user".to_string(),
|
|
scopes_enabled: true,
|
|
};
|
|
|
|
for path in [
|
|
"/openshell.v1.OpenShell/ReportPolicyStatus",
|
|
"/openshell.v1.OpenShell/PushSandboxLogs",
|
|
"/openshell.v1.OpenShell/SubmitPolicyAnalysis",
|
|
"/openshell.v1.OpenShell/GetSandboxProviderEnvironment",
|
|
"/openshell.v1.OpenShell/ConnectSupervisor",
|
|
"/openshell.v1.OpenShell/RelayStream",
|
|
"/openshell.v1.OpenShell/IssueSandboxToken",
|
|
"/openshell.v1.OpenShell/RefreshSandboxToken",
|
|
] {
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(Authentication::Accepted(
|
|
admin_user(),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), Some(policy.clone()));
|
|
|
|
let res = router.call(empty_request(path)).await.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_none(), "{path} reached handler");
|
|
// grpc-status=7 (PERMISSION_DENIED).
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("7"), "{path}");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn provider_write_scope_allows_update_without_provider_read() {
|
|
let policy = AuthzPolicy {
|
|
admin_role: "openshell-admin".to_string(),
|
|
user_role: "openshell-user".to_string(),
|
|
scopes_enabled: true,
|
|
};
|
|
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(Authentication::Accepted(
|
|
provider_writer_principal(),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), Some(policy.clone()));
|
|
let response = router
|
|
.call(empty_request(UPDATE_PROVIDER_PATH))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(response.status(), StatusCode::OK);
|
|
assert!(seen.lock().unwrap().is_some());
|
|
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(Authentication::Accepted(
|
|
provider_writer_principal(),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), Some(policy));
|
|
let response = router
|
|
.call(empty_request("/openshell.v1.OpenShell/GetProvider"))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(grpc_status(&response).as_deref(), Some("7"));
|
|
assert!(seen.lock().unwrap().is_none());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn sandbox_principal_is_denied_on_user_and_admin_methods() {
|
|
for path in [
|
|
"/openshell.v1.OpenShell/ListSandboxes",
|
|
"/openshell.v1.OpenShell/DeleteSandbox",
|
|
"/openshell.v1.OpenShell/CreateProvider",
|
|
"/openshell.v1.OpenShell/ApproveDraftChunk",
|
|
] {
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(Authentication::Accepted(
|
|
sandbox("sandbox-a"),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
|
|
let res = router.call(empty_request(path)).await.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_none(), "{path} reached handler");
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("7"), "{path}");
|
|
}
|
|
}
|
|
|
|
fn assert_authorization_denial(event: &serde_json::Value, operation: &str, reason: &str) {
|
|
assert_denial(
|
|
event,
|
|
&format!("/openshell.v1.OpenShell/{operation}"),
|
|
reason,
|
|
);
|
|
assert_eq!(event["api"]["operation"], operation);
|
|
assert_eq!(event["api"]["service"]["name"], "openshell.v1.OpenShell");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn sandbox_principal_on_admin_method_emits_denial_and_correlated_finding() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Accepted(sandbox("sandbox-a")),
|
|
)))]);
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let request = Request::builder()
|
|
.uri("/openshell.v1.OpenShell/CreateProvider")
|
|
.header("authorization", "Bearer canary-secret-token-9f2c")
|
|
.header("x-request-id", REQUEST_ID)
|
|
.body(Full::new(Bytes::new()))
|
|
.unwrap();
|
|
|
|
let res = router.call(request).await.unwrap();
|
|
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("7"));
|
|
let events = authorization_events(&capture);
|
|
assert_eq!(events.len(), 2, "{events:?}");
|
|
assert_authorization_denial(&events[0], "CreateProvider", "method_not_permitted");
|
|
assert_eq!(
|
|
events[0]["actor"],
|
|
serde_json::json!({"user": {"uid": "sandbox-a", "type_id": 99, "type": "Sandbox"}})
|
|
);
|
|
assert_eq!(events[0]["severity_id"], 3);
|
|
|
|
let finding = &events[1];
|
|
assert_boundary_finding(&events[0], finding);
|
|
for event in &events {
|
|
assert_eq!(event["metadata"]["correlation_uid"], REQUEST_ID);
|
|
assert_redacted(event, "canary-secret-token");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn user_missing_rbac_role_emits_denial_without_finding() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Accepted(user("alice")),
|
|
)))]);
|
|
let policy = AuthzPolicy {
|
|
admin_role: "openshell-admin".to_string(),
|
|
user_role: "openshell-user".to_string(),
|
|
scopes_enabled: false,
|
|
};
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), Some(policy));
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_none());
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("7"));
|
|
let events = authorization_events(&capture);
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_authorization_denial(&events[0], "ListSandboxes", "missing_role");
|
|
assert_eq!(
|
|
events[0]["actor"],
|
|
serde_json::json!({"user": {"uid": "alice", "type_id": 1, "type": "User"}})
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn sandbox_principal_on_unknown_method_is_denied_without_a_finding() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Accepted(sandbox("sandbox-a")),
|
|
)))]);
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/SomeFutureMethod"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("7"));
|
|
let events = authorization_events(&capture);
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_authorization_denial(&events[0], "SomeFutureMethod", "unknown_method");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn permitted_sandbox_call_emits_no_authorization_event() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Accepted(sandbox("sandbox-a")),
|
|
)))]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
|
|
router
|
|
.call(empty_request("/openshell.v1.OpenShell/GetSandboxConfig"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_some());
|
|
assert!(authorization_events(&capture).is_empty());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn peer_and_operator_denials_name_their_principal_type() {
|
|
for (principal_type, router) in [
|
|
(
|
|
"Admin",
|
|
AuthGrpcRouter::with_peer_identity(
|
|
PrincipalRecorder::new().0,
|
|
None,
|
|
None,
|
|
None,
|
|
true,
|
|
false,
|
|
)
|
|
.with_operator(Some(operator_principal())),
|
|
),
|
|
(
|
|
"Gateway Peer",
|
|
AuthGrpcRouter::new(
|
|
PrincipalRecorder::new().0,
|
|
Some(AuthenticatorChain::new(vec![Arc::new(
|
|
MockAuthenticator::returning(Ok(Authentication::Accepted(
|
|
Principal::Peer(crate::auth::principal::PeerPrincipal {
|
|
replica_id: "gateway-1".to_string(),
|
|
pod_uid: "pod-1".to_string(),
|
|
}),
|
|
))),
|
|
)])),
|
|
None,
|
|
),
|
|
),
|
|
] {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let mut router = router;
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ConnectSupervisor"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("7"), "{principal_type}");
|
|
let events = authorization_events(&capture);
|
|
assert_eq!(events.len(), 1, "{principal_type}: {events:?}");
|
|
assert_authorization_denial(
|
|
&events[0],
|
|
"ConnectSupervisor",
|
|
"method_not_permitted",
|
|
);
|
|
assert_eq!(events[0]["actor"]["user"]["type"], principal_type);
|
|
}
|
|
}
|
|
|
|
/// Calls the sandbox-scope guard for another sandbox, as a handler
|
|
/// would for a request body naming it.
|
|
#[derive(Clone)]
|
|
struct CrossSandboxHandler;
|
|
|
|
impl<B: Send + 'static> Service<Request<B>> for CrossSandboxHandler {
|
|
type Response = Response<tonic::body::Body>;
|
|
type Error = Infallible;
|
|
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
|
|
|
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
|
Poll::Ready(Ok(()))
|
|
}
|
|
|
|
fn call(&mut self, req: Request<B>) -> Self::Future {
|
|
let principal = req.extensions().get::<Principal>().cloned().unwrap();
|
|
Box::pin(async move {
|
|
let status = crate::auth::guard::ensure_sandbox_scope(&principal, "sandbox-b")
|
|
.expect_err("another sandbox must be denied");
|
|
Ok(status.into_http())
|
|
})
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn handler_denial_is_recorded_against_the_routed_request() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Accepted(sandbox("sandbox-a")),
|
|
)))]);
|
|
let mut router = AuthGrpcRouter::new(CrossSandboxHandler, Some(chain), None)
|
|
.with_connection_source(ConnectionSource::Peer("10.42.0.7:51234".parse().unwrap()));
|
|
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/GetSandboxConfig"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("7"));
|
|
let events = authorization_events(&capture);
|
|
assert_eq!(events.len(), 2, "{events:?}");
|
|
assert_authorization_denial(&events[0], "GetSandboxConfig", "cross_sandbox");
|
|
assert_eq!(events[0]["src_endpoint"]["ip"], "10.42.0.7");
|
|
assert_eq!(
|
|
events[0]["resources"],
|
|
serde_json::json!([{"uid": "sandbox-b", "type": "sandbox"}])
|
|
);
|
|
assert_boundary_finding(&events[0], &events[1]);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn missing_principal_returns_unauthenticated() {
|
|
let mock = Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::NotApplicable,
|
|
)));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
assert!(seen.lock().unwrap().is_none());
|
|
// tonic sets grpc-status=16 (UNAUTHENTICATED) in trailers.
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("16"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn legacy_sandbox_jwt_cannot_reach_sensitive_sandbox_methods() {
|
|
let material = openshell_bootstrap::jwt::generate_jwt_key().expect("JWT key");
|
|
let authority = Arc::new(
|
|
crate::auth::sandbox_jwt::SandboxSessionJwtAuthority::from_pem(
|
|
material.signing_key_pem.as_bytes(),
|
|
material.public_key_pem.as_bytes(),
|
|
material.kid.clone(),
|
|
"test",
|
|
Some(Duration::from_mins(15)),
|
|
)
|
|
.expect("session authority"),
|
|
);
|
|
let store = Arc::new(
|
|
crate::persistence::Store::connect("sqlite::memory:")
|
|
.await
|
|
.expect("store"),
|
|
);
|
|
let token = legacy_sandbox_token(&material);
|
|
|
|
for path in [
|
|
"/openshell.v1.OpenShell/GetSandboxProviderEnvironment",
|
|
"/openshell.v1.OpenShell/ConnectSupervisor",
|
|
] {
|
|
let authenticator = Arc::new(
|
|
crate::auth::sandbox_jwt::SandboxSessionJwtAuthenticator::new(
|
|
authority.clone(),
|
|
store.clone(),
|
|
),
|
|
);
|
|
let chain = AuthenticatorChain::new(vec![authenticator]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let request = Request::builder()
|
|
.uri(path)
|
|
.header("authorization", format!("Bearer {token}"))
|
|
.body(Full::new(Bytes::new()))
|
|
.expect("request");
|
|
|
|
let response = router.call(request).await.expect("router response");
|
|
|
|
assert!(seen.lock().unwrap().is_none(), "{path} reached handler");
|
|
assert_eq!(grpc_status(&response).as_deref(), Some("16"), "{path}");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn authenticator_error_short_circuits() {
|
|
let mock = Arc::new(MockAuthenticator::returning(Err(AuthRejection::new(
|
|
AuthMechanism::Oidc,
|
|
AuthFailureReason::InvalidCredential,
|
|
tonic::Status::unauthenticated("forged"),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![mock]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
assert!(seen.lock().unwrap().is_none());
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("16"));
|
|
}
|
|
|
|
/// Captured events other than authentication events.
|
|
fn authorization_events(capture: &OcsfCapture) -> Vec<serde_json::Value> {
|
|
capture
|
|
.json_events()
|
|
.into_iter()
|
|
.filter(|event| event["class_uid"] != 3002)
|
|
.collect()
|
|
}
|
|
|
|
/// Assert that `event` is the router's unidentified Authentication
|
|
/// failure for `reason`, at `severity_id`.
|
|
fn assert_router_authentication_failure(
|
|
event: &serde_json::Value,
|
|
path: &str,
|
|
reason: &str,
|
|
severity_id: u8,
|
|
) {
|
|
assert_authentication_failure(event, path, reason);
|
|
assert_eq!(event["severity_id"], severity_id);
|
|
assert_eq!(event["user"]["type_id"], 0);
|
|
assert_eq!(event["metadata"]["product"]["name"], "OpenShell Gateway");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn missing_credential_emits_authentication_failure() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::NotApplicable,
|
|
)))]);
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let path = "/openshell.v1.OpenShell/ListSandboxes";
|
|
|
|
let res = router.call(empty_request(path)).await.unwrap();
|
|
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("16"));
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_router_authentication_failure(&events[0], path, "missing_credential", 2);
|
|
assert!(events[0].get("auth_protocol_id").is_none());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn missing_client_certificate_emits_mtls_authentication_failure() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router =
|
|
AuthGrpcRouter::with_peer_identity(recorder, None, None, None, true, false);
|
|
let path = "/openshell.v1.OpenShell/ListSandboxes";
|
|
|
|
let res = router.call(empty_request(path)).await.unwrap();
|
|
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("16"));
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_router_authentication_failure(&events[0], path, "missing_credential", 2);
|
|
assert_eq!(events[0]["auth_protocol_id"], 99);
|
|
assert_eq!(events[0]["auth_protocol"], "mtls");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn operator_with_authorization_header_emits_conflicting_credentials() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router =
|
|
AuthGrpcRouter::with_peer_identity(recorder, None, None, None, true, false)
|
|
.with_operator(Some(operator_principal()));
|
|
let path = "/openshell.v1.OpenShell/ListSandboxes";
|
|
let canary = "Bearer canary-secret-token-9f2c";
|
|
let request = Request::builder()
|
|
.uri(path)
|
|
.header("authorization", canary)
|
|
.body(Full::new(Bytes::new()))
|
|
.unwrap();
|
|
|
|
let res = router.call(request).await.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_none());
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("16"));
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_router_authentication_failure(&events[0], path, "conflicting_credentials", 3);
|
|
assert_eq!(events[0]["auth_protocol"], "mtls");
|
|
assert_redacted(&events[0], "canary-secret-token");
|
|
}
|
|
|
|
async fn call_with_rejection(
|
|
rejection: AuthRejection,
|
|
path: &str,
|
|
) -> (tonic::Status, Vec<serde_json::Value>) {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Err(
|
|
rejection,
|
|
)))]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let request = Request::builder()
|
|
.uri(path)
|
|
.header("authorization", "Bearer canary-secret-token-9f2c")
|
|
.body(Full::new(Bytes::new()))
|
|
.unwrap();
|
|
|
|
let res = router.call(request).await.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_none());
|
|
let status = tonic::Status::from_header_map(res.headers()).expect("grpc status");
|
|
(status, capture.json_events())
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn declined_credential_emits_its_classification_with_the_missing_credential_status() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Declined(crate::auth::authenticator::AuthDecline {
|
|
mechanism: AuthMechanism::GatewayPeer,
|
|
reason: AuthFailureReason::InvalidCredential,
|
|
}),
|
|
)))]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let path = "/openshell.v1.OpenShell/PeerRelay";
|
|
|
|
let res = router.call(empty_request(path)).await.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_none());
|
|
let status = tonic::Status::from_header_map(res.headers()).expect("grpc status");
|
|
assert_eq!(status.code(), tonic::Code::Unauthenticated);
|
|
assert_eq!(status.message(), "missing authorization header");
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_router_authentication_failure(&events[0], path, "invalid_credential", 3);
|
|
assert_eq!(events[0]["auth_protocol"], "k8s_service_account");
|
|
}
|
|
|
|
async fn missing_credential_event(
|
|
source: ConnectionSource,
|
|
request_id: &str,
|
|
) -> serde_json::Value {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::NotApplicable,
|
|
)))]);
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router =
|
|
AuthGrpcRouter::new(recorder, Some(chain), None).with_connection_source(source);
|
|
let request = Request::builder()
|
|
.uri("/openshell.v1.OpenShell/ListSandboxes")
|
|
.header("x-request-id", request_id)
|
|
.body(Full::new(Bytes::new()))
|
|
.unwrap();
|
|
|
|
router.call(request).await.unwrap();
|
|
|
|
let mut events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
events.remove(0)
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn authentication_failure_carries_peer_address_and_request_id() {
|
|
let event = missing_credential_event(peer(), REQUEST_ID).await;
|
|
|
|
assert_eq!(
|
|
event["src_endpoint"],
|
|
serde_json::json!({"ip": PEER_IP, "port": 51234})
|
|
);
|
|
assert_eq!(event["metadata"]["correlation_uid"], REQUEST_ID);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn ipv4_mapped_peer_address_is_reported_as_ipv4() {
|
|
let event = missing_credential_event(
|
|
ConnectionSource::Peer("[::ffff:203.0.113.7]:51234".parse().unwrap()),
|
|
REQUEST_ID,
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(event["src_endpoint"]["ip"], PEER_IP);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn tunnel_source_is_named_and_non_uuid_request_ids_are_dropped() {
|
|
let event =
|
|
missing_credential_event(ConnectionSource::EdgeTunnel, "attacker [controlled]")
|
|
.await;
|
|
|
|
assert_eq!(
|
|
event["src_endpoint"],
|
|
serde_json::json!({"name": "edge-tunnel"})
|
|
);
|
|
assert!(event["metadata"].get("correlation_uid").is_none());
|
|
assert_redacted(&event, "attacker");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn connection_source_reaches_handlers() {
|
|
let seen = Arc::new(Mutex::new(None));
|
|
let recorded = seen.clone();
|
|
let inner = tower::service_fn(move |req: Request<Full<Bytes>>| {
|
|
*recorded.lock().unwrap() = req.extensions().get::<ConnectionSource>().copied();
|
|
async { Ok::<_, Infallible>(Response::new(tonic::body::Body::empty())) }
|
|
});
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Accepted(user("alice")),
|
|
)))]);
|
|
let source = peer();
|
|
let mut router =
|
|
AuthGrpcRouter::new(inner, Some(chain), None).with_connection_source(source);
|
|
|
|
router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(*seen.lock().unwrap(), Some(source));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn rejected_oidc_credential_emits_classified_authentication_failure() {
|
|
let path = "/openshell.v1.OpenShell/ListSandboxes";
|
|
let (status, events) = call_with_rejection(
|
|
AuthRejection::new(
|
|
AuthMechanism::Oidc,
|
|
AuthFailureReason::ExpiredCredential,
|
|
tonic::Status::unauthenticated("invalid token: ExpiredSignature"),
|
|
),
|
|
path,
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(status.code(), tonic::Code::Unauthenticated);
|
|
assert_eq!(status.message(), "invalid token: ExpiredSignature");
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_router_authentication_failure(&events[0], path, "expired_credential", 2);
|
|
assert_eq!(events[0]["auth_protocol_id"], 4);
|
|
assert_eq!(events[0]["auth_protocol"], "OpenID");
|
|
assert_redacted(&events[0], "canary-secret-token");
|
|
assert_redacted(&events[0], "ExpiredSignature");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn rejected_sandbox_credential_emits_classified_authentication_failure() {
|
|
let path = "/openshell.v1.OpenShell/GetSandboxProviderEnvironment";
|
|
let (status, events) = call_with_rejection(
|
|
AuthRejection::new(
|
|
AuthMechanism::SandboxJwt,
|
|
AuthFailureReason::StaleIdentity,
|
|
tonic::Status::unauthenticated(
|
|
"gateway token does not match the active sandbox identity",
|
|
),
|
|
),
|
|
path,
|
|
)
|
|
.await;
|
|
|
|
assert_eq!(status.code(), tonic::Code::Unauthenticated);
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_router_authentication_failure(&events[0], path, "stale_identity", 3);
|
|
assert_eq!(events[0]["auth_protocol_id"], 99);
|
|
assert_eq!(events[0]["auth_protocol"], "sandbox_jwt");
|
|
}
|
|
|
|
fn bearer_request(path: &str, token: &str, request_id: &str) -> Request<Full<Bytes>> {
|
|
Request::builder()
|
|
.uri(path)
|
|
.header("authorization", format!("Bearer {token}"))
|
|
.header("x-request-id", request_id)
|
|
.body(Full::new(Bytes::new()))
|
|
.unwrap()
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn credential_logs_on_once_and_exempt_requests_emit_nothing() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Accepted(user("alice")),
|
|
)))]);
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router =
|
|
AuthGrpcRouter::new(recorder, Some(chain), None).with_connection_source(peer());
|
|
let path = "/openshell.v1.OpenShell/ListSandboxes";
|
|
|
|
for request in [
|
|
bearer_request(path, "canary-secret-token-9f2c", REQUEST_ID),
|
|
bearer_request(
|
|
path,
|
|
"canary-secret-token-9f2c",
|
|
"0d9a4c1e-5b6f-4a7d-9e8c-1f2a3b4c5d6e",
|
|
),
|
|
bearer_request(
|
|
"/openshell.v1.OpenShell/Health",
|
|
"canary-secret-token-9f2c",
|
|
REQUEST_ID,
|
|
),
|
|
] {
|
|
let res = router.call(request).await.unwrap();
|
|
assert!(grpc_status(&res).is_none());
|
|
}
|
|
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_logon(&events[0], path);
|
|
assert_eq!(
|
|
events[0]["user"],
|
|
serde_json::json!({"uid": "alice", "type_id": 1, "type": "User"})
|
|
);
|
|
assert_eq!(events[0]["auth_protocol_id"], 4);
|
|
assert_eq!(
|
|
events[0]["src_endpoint"],
|
|
serde_json::json!({"ip": PEER_IP, "port": 51234})
|
|
);
|
|
assert_eq!(events[0]["metadata"]["correlation_uid"], REQUEST_ID);
|
|
assert_redacted(&events[0], "canary-secret-token");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn each_new_credential_logs_on() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Accepted(sandbox("sandbox-a")),
|
|
)))]);
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let path = "/openshell.v1.OpenShell/GetSandboxConfig";
|
|
|
|
for token in ["first-token", "refreshed-token", "first-token"] {
|
|
router
|
|
.call(bearer_request(path, token, REQUEST_ID))
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 2, "{events:?}");
|
|
for event in &events {
|
|
assert_logon(event, path);
|
|
assert_eq!(
|
|
event["user"],
|
|
serde_json::json!({"uid": "sandbox-a", "type_id": 99, "type": "Sandbox"})
|
|
);
|
|
assert_eq!(event["auth_protocol"], "sandbox_jwt");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unverified_header_bytes_do_not_make_a_new_logon() {
|
|
let path = "/openshell.v1.OpenShell/ListSandboxes";
|
|
for (case, router, headers) in [
|
|
(
|
|
"mTLS user with ignored authorization headers",
|
|
AuthGrpcRouter::with_peer_identity(
|
|
PrincipalRecorder::new().0,
|
|
None,
|
|
None,
|
|
Some(mtls_identity("alice")),
|
|
true,
|
|
false,
|
|
),
|
|
["Basic first-nonce", "Basic second-nonce"],
|
|
),
|
|
(
|
|
"bearer scheme spelled differently",
|
|
AuthGrpcRouter::new(
|
|
PrincipalRecorder::new().0,
|
|
Some(AuthenticatorChain::new(vec![Arc::new(
|
|
MockAuthenticator::returning(Ok(Authentication::Accepted(user(
|
|
"alice",
|
|
)))),
|
|
)])),
|
|
None,
|
|
),
|
|
["Bearer alice-token", "bEARER alice-token"],
|
|
),
|
|
] {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let mut router = router;
|
|
|
|
for authorization in headers {
|
|
let request = Request::builder()
|
|
.uri(path)
|
|
.header("authorization", authorization)
|
|
.body(Full::new(Bytes::new()))
|
|
.unwrap();
|
|
router.call(request).await.unwrap();
|
|
}
|
|
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{case}: {events:?}");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn logon_names_the_protocol_that_established_the_principal() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(Ok(
|
|
Authentication::Accepted(Principal::Peer(crate::auth::principal::PeerPrincipal {
|
|
replica_id: "gateway-1".to_string(),
|
|
pod_uid: "pod-1".to_string(),
|
|
})),
|
|
)))]);
|
|
let mut router = AuthGrpcRouter::new(PrincipalRecorder::new().0, Some(chain), None);
|
|
let path = "/openshell.v1.OpenShell/PeerRelay";
|
|
|
|
router
|
|
.call(bearer_request(path, "peer-token", REQUEST_ID))
|
|
.await
|
|
.unwrap();
|
|
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
assert_logon(&events[0], path);
|
|
assert_eq!(events[0]["user"]["type"], "Gateway Peer");
|
|
assert_eq!(events[0]["auth_protocol"], "k8s_service_account");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn verified_credentials_log_on_before_a_sandbox_token_method_denies_them() {
|
|
for path in [
|
|
"/openshell.v1.OpenShell/IssueSandboxToken",
|
|
"/openshell.v1.OpenShell/RefreshSandboxToken",
|
|
] {
|
|
for (principal, principal_type) in [
|
|
(user("alice"), "User"),
|
|
(Principal::Operator(operator_principal()), "Admin"),
|
|
(
|
|
Principal::Peer(crate::auth::principal::PeerPrincipal {
|
|
replica_id: "gateway-1".to_string(),
|
|
pod_uid: "pod-1".to_string(),
|
|
}),
|
|
"Gateway Peer",
|
|
),
|
|
] {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(
|
|
MockAuthenticator::returning(Ok(Authentication::Accepted(principal))),
|
|
)]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
|
|
let res = router
|
|
.call(bearer_request(path, "verified-token", REQUEST_ID))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert_eq!(grpc_status(&res).as_deref(), Some("7"), "{path}");
|
|
assert!(seen.lock().unwrap().is_none(), "{path}");
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 2, "{principal_type} {path}: {events:?}");
|
|
assert_logon(&events[0], path);
|
|
assert_eq!(events[0]["user"]["type"], principal_type);
|
|
assert_eq!(events[1]["class_uid"], 6003);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn handlers_that_finish_verification_record_their_own_logon() {
|
|
for (principal, path) in [
|
|
(
|
|
Principal::Sandbox(SandboxPrincipal {
|
|
sandbox_id: "sandbox-a".to_string(),
|
|
source: SandboxIdentitySource::ComputeDriver {
|
|
driver_name: "kubernetes".to_string(),
|
|
runtime_identity: "pod-uid".to_string(),
|
|
},
|
|
trust_domain: None,
|
|
}),
|
|
"/openshell.v1.OpenShell/IssueSandboxToken",
|
|
),
|
|
(
|
|
Principal::Sandbox(SandboxPrincipal {
|
|
sandbox_id: "sandbox-a".to_string(),
|
|
source: SandboxIdentitySource::BootstrapJwt {
|
|
issuer: "openshell-gateway:test".to_string(),
|
|
},
|
|
trust_domain: None,
|
|
}),
|
|
"/openshell.v1.OpenShell/RefreshSandboxToken",
|
|
),
|
|
] {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(
|
|
Ok(Authentication::Accepted(principal)),
|
|
))]);
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
|
|
router
|
|
.call(bearer_request(path, "sandbox-token", REQUEST_ID))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_some(), "{path} reached the handler");
|
|
let events = capture.json_events();
|
|
assert!(events.is_empty(), "{path}: {events:?}");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn connections_share_logons() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let logons = audit::Logons::default();
|
|
let path = "/openshell.v1.OpenShell/ListSandboxes";
|
|
|
|
for _ in 0..2 {
|
|
let chain = AuthenticatorChain::new(vec![Arc::new(MockAuthenticator::returning(
|
|
Ok(Authentication::Accepted(user("alice"))),
|
|
))]);
|
|
let mut connection =
|
|
AuthGrpcRouter::new(PrincipalRecorder::new().0, Some(chain), None)
|
|
.with_logons(logons.clone());
|
|
connection
|
|
.call(bearer_request(path, "alice-token", REQUEST_ID))
|
|
.await
|
|
.unwrap();
|
|
}
|
|
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 1, "{events:?}");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn each_certificate_logs_on_once() {
|
|
// A connection authenticated by the certificate with `fingerprint`,
|
|
// whose subject is always "alice".
|
|
let user = |fingerprint: &str, logons: &audit::Logons| {
|
|
AuthGrpcRouter::with_peer_identity(
|
|
PrincipalRecorder::new().0,
|
|
None,
|
|
None,
|
|
Some(mtls_identity("alice")),
|
|
true,
|
|
false,
|
|
)
|
|
.with_peer_certificate(Some(fingerprint.to_string()))
|
|
.with_logons(logons.clone())
|
|
};
|
|
let admin = |fingerprint: &str, logons: &audit::Logons| {
|
|
AuthGrpcRouter::with_peer_identity(
|
|
PrincipalRecorder::new().0,
|
|
None,
|
|
None,
|
|
None,
|
|
true,
|
|
false,
|
|
)
|
|
.with_operator(Some(OperatorPrincipal {
|
|
certificate_sha256: fingerprint.to_string(),
|
|
..operator_principal()
|
|
}))
|
|
.with_logons(logons.clone())
|
|
};
|
|
for principal_type in ["User", "Admin"] {
|
|
let router = |fingerprint: &str, logons: &audit::Logons| {
|
|
if principal_type == "User" {
|
|
user(fingerprint, logons)
|
|
} else {
|
|
admin(fingerprint, logons)
|
|
}
|
|
};
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let logons = audit::Logons::default();
|
|
let path = "/openshell.v1.OpenShell/ListSandboxes";
|
|
|
|
// Two connections with one certificate, then a renewed
|
|
// certificate for the same subject.
|
|
for fingerprint in ["sha256-original", "sha256-original", "sha256-renewed"] {
|
|
let mut router = router(fingerprint, &logons);
|
|
for _ in 0..2 {
|
|
router.call(empty_request(path)).await.unwrap();
|
|
}
|
|
}
|
|
|
|
let events = capture.json_events();
|
|
assert_eq!(events.len(), 2, "{principal_type}: {events:?}");
|
|
for event in &events {
|
|
assert_logon(event, path);
|
|
assert_eq!(event["user"]["type"], principal_type);
|
|
assert_eq!(event["auth_protocol"], "mtls");
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unauthenticated_dev_user_does_not_log_on() {
|
|
let capture = OcsfCapture::default();
|
|
let _guard = capture.install();
|
|
let (recorder, seen) = PrincipalRecorder::new();
|
|
let mut router =
|
|
AuthGrpcRouter::with_peer_identity(recorder, None, None, None, false, true);
|
|
|
|
router
|
|
.call(empty_request("/openshell.v1.OpenShell/ListSandboxes"))
|
|
.await
|
|
.unwrap();
|
|
|
|
assert!(seen.lock().unwrap().is_some());
|
|
let events = capture.json_events();
|
|
assert!(events.is_empty(), "{events:?}");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn health_methods_bypass_chain() {
|
|
// Authenticator is wired to fail-closed; the request still gets
|
|
// through because the path is exempt.
|
|
let mock = Arc::new(MockAuthenticator::returning(Err(AuthRejection::new(
|
|
AuthMechanism::Oidc,
|
|
AuthFailureReason::InvalidCredential,
|
|
tonic::Status::unauthenticated("would reject"),
|
|
))));
|
|
let chain = AuthenticatorChain::new(vec![mock.clone()]);
|
|
let (recorder, _) = PrincipalRecorder::new();
|
|
let mut router = AuthGrpcRouter::new(recorder, Some(chain), None);
|
|
let res = router
|
|
.call(empty_request("/openshell.v1.OpenShell/Health"))
|
|
.await
|
|
.unwrap();
|
|
assert_eq!(res.status(), 200);
|
|
assert_eq!(mock.call_count(), 0, "health must not consult the chain");
|
|
}
|
|
}
|
|
}
|