mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-11-05 08:36:09 +08:00
* feat(extensions)!: normalize protocol negotiation Closes #3057 Introduce a shared extension handshake, enforce protocol and capability compatibility across extension families, and expose immutable negotiated snapshots through gateway info and the Go SDK. Signed-off-by: Seth Jennings <sjenning@redhat.com> * fix(credentials): fail fast on negotiation errors Signed-off-by: Seth Jennings <sjenning@redhat.com> * fix(extensions): validate gateway handshake metadata Signed-off-by: Seth Jennings <sjenning@redhat.com> * fix(go-sdk): re-export extension kind constants Signed-off-by: Seth Jennings <sjenning@redhat.com> * fix(extensions): fail fast on credential handshake rejection Signed-off-by: Seth Jennings <sjenning@redhat.com> --------- Signed-off-by: Seth Jennings <sjenning@redhat.com>
6316 lines
242 KiB
Rust
6316 lines
242 KiB
Rust
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
// SPDX-License-Identifier: Apache-2.0
|
|
|
|
//! Supervisor middleware registration and chain execution.
|
|
|
|
pub mod headers;
|
|
mod remote;
|
|
mod response;
|
|
mod websocket;
|
|
|
|
pub use response::{
|
|
HttpResponseDiagnostics, HttpResponseFinish, HttpResponseInvocation,
|
|
HttpResponseInvocationOutcome, HttpResponseMiddlewareFailure, HttpResponsePreflightInput,
|
|
HttpResponsePreflightOutcome, HttpResponseSession, MAX_HTTP_RESPONSE_RETAINED_BODY_BYTES,
|
|
MAX_HTTP_RESPONSE_STREAM_UNIT_BYTES, is_stale_http_response_integrity_header,
|
|
};
|
|
|
|
pub use websocket::{
|
|
WebSocketCoverage, WebSocketCoverageState, WebSocketInvocation, WebSocketInvocationOutcome,
|
|
WebSocketMessageAdmission, WebSocketMessageOutcome, WebSocketMessageType,
|
|
WebSocketPreflightInput, WebSocketPreflightResult, WebSocketSession,
|
|
WebSocketSessionStartOutcome,
|
|
};
|
|
|
|
use std::collections::{BTreeMap, HashMap, HashSet};
|
|
use std::future::Future;
|
|
use std::sync::Arc;
|
|
use std::time::Duration;
|
|
|
|
use miette::{Result, miette};
|
|
use prost::Message;
|
|
|
|
use openshell_core::extension_protocol::{
|
|
ExtensionFamily, NegotiatedExtension, gateway_metadata, negotiate,
|
|
};
|
|
use openshell_core::proto::middleware::v1::supervisor_middleware_server::SupervisorMiddleware;
|
|
use openshell_core::proto::{
|
|
Decision, Finding, HeaderMutation, HttpHeader, HttpRequestEvaluation, HttpRequestTarget,
|
|
MiddlewareBinding, MiddlewareDescribeRequest, MiddlewareManifest, NetworkMiddlewareConfig,
|
|
RequestContext, SandboxPolicy, SupervisorMiddlewareOperation, SupervisorMiddlewarePhase,
|
|
SupervisorMiddlewareService, ValidateConfigRequest, ValidateConfigResponse,
|
|
};
|
|
use tokio::sync::{OnceCell, OwnedSemaphorePermit, Semaphore};
|
|
use tonic::{Request, Response as TonicResponse, Status as TonicStatus};
|
|
|
|
pub use openshell_core::middleware::{
|
|
HttpRequestView, HttpResponseResultStream, InProcessMiddleware, SupervisorMiddlewareEndpoint,
|
|
WebSocketResponseStream,
|
|
};
|
|
pub type MiddlewareService =
|
|
dyn SupervisorMiddleware<EvaluateWebSocketSessionStream = WebSocketResponseStream>;
|
|
|
|
struct GeneratedMiddlewareEndpoint {
|
|
service: Arc<MiddlewareService>,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl SupervisorMiddlewareEndpoint for GeneratedMiddlewareEndpoint {
|
|
async fn describe(
|
|
&self,
|
|
request: Request<MiddlewareDescribeRequest>,
|
|
) -> std::result::Result<TonicResponse<MiddlewareManifest>, TonicStatus> {
|
|
self.service.describe(request).await
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
request: Request<ValidateConfigRequest>,
|
|
) -> std::result::Result<TonicResponse<ValidateConfigResponse>, TonicStatus> {
|
|
self.service.validate_config(request).await
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: Request<HttpRequestEvaluation>,
|
|
) -> std::result::Result<TonicResponse<openshell_core::proto::HttpRequestResult>, TonicStatus>
|
|
{
|
|
self.service.evaluate_http_request(request).await
|
|
}
|
|
|
|
async fn open_websocket_session(
|
|
&self,
|
|
_receiver: tokio::sync::mpsc::Receiver<openshell_core::proto::WebSocketSessionEvent>,
|
|
) -> std::result::Result<WebSocketResponseStream, TonicStatus> {
|
|
Err(TonicStatus::unimplemented(
|
|
"middleware service does not expose an in-process WebSocket stream",
|
|
))
|
|
}
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl InProcessMiddleware for GeneratedMiddlewareEndpoint {
|
|
async fn describe(&self) -> MiddlewareManifest {
|
|
self.service
|
|
.describe(Request::new(MiddlewareDescribeRequest {
|
|
gateway: Some(gateway_metadata(ExtensionFamily::SupervisorMiddleware)),
|
|
}))
|
|
.await
|
|
.expect("generated in-process Describe failed")
|
|
.into_inner()
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
middleware_name: &str,
|
|
config: &prost_types::Struct,
|
|
) -> Result<()> {
|
|
let response = self
|
|
.service
|
|
.validate_config(Request::new(ValidateConfigRequest {
|
|
config: Some(config.clone()),
|
|
middleware_name: middleware_name.to_string(),
|
|
}))
|
|
.await
|
|
.map_err(|error| miette!("{error}"))?
|
|
.into_inner();
|
|
if response.valid {
|
|
Ok(())
|
|
} else {
|
|
Err(miette!("{}", response.reason))
|
|
}
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: HttpRequestView<'_>,
|
|
) -> Result<openshell_core::proto::HttpRequestResult> {
|
|
self.service
|
|
.evaluate_http_request(Request::new(request_view_to_evaluation(request)))
|
|
.await
|
|
.map(tonic::Response::into_inner)
|
|
.map_err(|error| miette!("{error}"))
|
|
}
|
|
}
|
|
|
|
/// Adapt a generated HTTP-only service to the borrowed in-process contract.
|
|
///
|
|
/// This compatibility adapter is intended for tests and downstream HTTP-only
|
|
/// implementations. First-party built-ins implement [`InProcessMiddleware`]
|
|
/// directly so their HTTP path remains allocation-free.
|
|
pub fn http_only_endpoint(service: Arc<MiddlewareService>) -> Arc<dyn InProcessMiddleware> {
|
|
Arc::new(GeneratedMiddlewareEndpoint { service })
|
|
}
|
|
|
|
struct EndpointInProcessAdapter {
|
|
endpoint: Arc<dyn SupervisorMiddlewareEndpoint>,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl InProcessMiddleware for EndpointInProcessAdapter {
|
|
async fn describe(&self) -> MiddlewareManifest {
|
|
self.endpoint
|
|
.describe(Request::new(MiddlewareDescribeRequest {
|
|
gateway: Some(gateway_metadata(ExtensionFamily::SupervisorMiddleware)),
|
|
}))
|
|
.await
|
|
.expect("in-process endpoint Describe failed")
|
|
.into_inner()
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
middleware_name: &str,
|
|
config: &prost_types::Struct,
|
|
) -> Result<()> {
|
|
let response = self
|
|
.endpoint
|
|
.validate_config(Request::new(ValidateConfigRequest {
|
|
config: Some(config.clone()),
|
|
middleware_name: middleware_name.to_string(),
|
|
}))
|
|
.await
|
|
.map_err(|error| miette!("{error}"))?
|
|
.into_inner();
|
|
if response.valid {
|
|
Ok(())
|
|
} else {
|
|
Err(miette!("{}", response.reason))
|
|
}
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: HttpRequestView<'_>,
|
|
) -> Result<openshell_core::proto::HttpRequestResult> {
|
|
self.endpoint
|
|
.evaluate_http_request(Request::new(request_view_to_evaluation(request)))
|
|
.await
|
|
.map(tonic::Response::into_inner)
|
|
.map_err(|error| miette!("{error}"))
|
|
}
|
|
|
|
async fn open_websocket_session(
|
|
&self,
|
|
requests: tokio::sync::mpsc::Receiver<openshell_core::proto::WebSocketSessionEvent>,
|
|
) -> std::result::Result<WebSocketResponseStream, tonic::Status> {
|
|
self.endpoint.open_websocket_session(requests).await
|
|
}
|
|
|
|
async fn open_http_response_pre_return(
|
|
&self,
|
|
requests: tokio::sync::mpsc::Receiver<openshell_core::proto::HttpResponseEvent>,
|
|
) -> std::result::Result<HttpResponseResultStream, tonic::Status> {
|
|
self.endpoint.open_http_response_pre_return(requests).await
|
|
}
|
|
}
|
|
|
|
/// Adapt a transport-neutral endpoint to the in-process registry contract.
|
|
///
|
|
/// Prefer implementing [`InProcessMiddleware`] directly. This compatibility
|
|
/// path materializes an owned HTTP request, but preserves direct WebSocket
|
|
/// streams for endpoint implementations that predate the borrowed contract.
|
|
pub fn in_process_endpoint(
|
|
endpoint: Arc<dyn SupervisorMiddlewareEndpoint>,
|
|
) -> Arc<dyn InProcessMiddleware> {
|
|
Arc::new(EndpointInProcessAdapter { endpoint })
|
|
}
|
|
|
|
/// Maximum short-lived middleware work items allowed to wait for active
|
|
/// capacity.
|
|
///
|
|
/// Waiters do not buffer request or message bodies, so the queue can absorb a
|
|
/// larger burst without increasing the active payload-memory bound.
|
|
pub const MAX_QUEUED_MIDDLEWARE_WORK: usize = MAX_CONCURRENT_MIDDLEWARE_WORK * 2;
|
|
|
|
/// One slot in the shared middleware work budget.
|
|
///
|
|
/// Callers that buffer request or message bodies acquire this guard first and
|
|
/// retain it through evaluation, bounding aggregate buffered middleware input.
|
|
#[derive(Debug)]
|
|
pub struct MiddlewareWorkAdmission {
|
|
_work: OwnedSemaphorePermit,
|
|
saturated: bool,
|
|
}
|
|
|
|
impl MiddlewareWorkAdmission {
|
|
pub fn saturated(&self) -> bool {
|
|
self.saturated
|
|
}
|
|
}
|
|
|
|
/// Result of attempting to enter the bounded middleware work queue.
|
|
///
|
|
/// Active-capacity saturation is ordinary backpressure: callers that obtain a
|
|
/// waiter slot eventually receive [`Self::Admitted`]. [`Self::QueueExhausted`]
|
|
/// is immediate load shedding after both active capacity and the waiter queue
|
|
/// are full.
|
|
#[derive(Debug)]
|
|
pub enum MiddlewareWorkAdmissionOutcome {
|
|
Admitted(MiddlewareWorkAdmission),
|
|
QueueExhausted,
|
|
}
|
|
|
|
impl MiddlewareWorkAdmissionOutcome {
|
|
/// Preserve the existing failure behavior for protocols whose outer layer
|
|
/// already translates middleware admission errors into a stable response
|
|
/// or typed termination.
|
|
pub fn into_admission(self) -> Result<MiddlewareWorkAdmission> {
|
|
match self {
|
|
Self::Admitted(admission) => Ok(admission),
|
|
Self::QueueExhausted => Err(miette!(
|
|
"middleware admission queue is full; refusing additional buffered work"
|
|
)),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// One slot in the shared persistent middleware session budget.
|
|
///
|
|
/// Protocol-specific session runners retain this guard while at least one
|
|
/// streaming stage remains active. Registry replacement preserves the shared
|
|
/// admission state so future streaming HTTP middleware can use the same
|
|
/// process-wide bound.
|
|
#[derive(Debug)]
|
|
struct MiddlewareSessionPermit {
|
|
_session: OwnedSemaphorePermit,
|
|
}
|
|
|
|
enum MiddlewareSessionAdmission {
|
|
Admitted(MiddlewareSessionPermit),
|
|
AtCapacity,
|
|
}
|
|
|
|
pub use openshell_core::middleware::{
|
|
DEFAULT_MIDDLEWARE_TIMEOUT, MAX_CONCURRENT_MIDDLEWARE_SESSIONS, MAX_CONCURRENT_MIDDLEWARE_WORK,
|
|
MAX_MIDDLEWARE_CHAIN_FINDINGS, MAX_MIDDLEWARE_CHAIN_STAGES, MAX_MIDDLEWARE_CHAIN_TIMEOUT,
|
|
MAX_MIDDLEWARE_CONFIGS, MAX_MIDDLEWARE_FINDINGS_PER_STAGE, MAX_MIDDLEWARE_PREFLIGHT_TIMEOUT,
|
|
MAX_MIDDLEWARE_SELECTOR_PATTERNS, MAX_MIDDLEWARE_TIMEOUT, MIN_MIDDLEWARE_TIMEOUT,
|
|
middleware_timeout_or_default, parse_middleware_timeout,
|
|
};
|
|
|
|
/// Largest logical payload or replacement accepted by the middleware platform.
|
|
pub const MAX_MIDDLEWARE_PAYLOAD_BYTES: usize = 4 * 1024 * 1024;
|
|
/// Largest encoded service-specific configuration attached to one evaluation.
|
|
pub const MAX_MIDDLEWARE_CONFIG_BYTES: usize = 64 * 1024;
|
|
/// Largest encoded request identity context attached to one evaluation.
|
|
pub const MAX_MIDDLEWARE_CONTEXT_BYTES: usize = 4 * 1024;
|
|
/// Largest encoded destination and request target attached to one evaluation.
|
|
pub const MAX_MIDDLEWARE_TARGET_BYTES: usize = 32 * 1024;
|
|
/// Largest number of request header lines exposed to one middleware.
|
|
pub const MAX_MIDDLEWARE_HEADERS: usize = 128;
|
|
/// Largest encoded request header collection exposed to one middleware.
|
|
pub const MAX_MIDDLEWARE_HEADER_BYTES: usize = 64 * 1024;
|
|
/// Largest operator-provided reason accepted in one middleware result.
|
|
pub const MAX_MIDDLEWARE_REASON_BYTES: usize = 4 * 1024;
|
|
/// Largest stable reason code accepted in one middleware result.
|
|
pub const MAX_MIDDLEWARE_REASON_CODE_BYTES: usize = 64;
|
|
/// Largest encoded individual finding accepted from one middleware stage.
|
|
pub const MAX_MIDDLEWARE_FINDING_BYTES: usize = 4 * 1024;
|
|
/// Largest number of metadata entries accepted from one middleware stage.
|
|
pub const MAX_MIDDLEWARE_METADATA_ENTRIES: usize = 64;
|
|
/// Largest combined metadata key/value payload accepted from one middleware stage.
|
|
pub const MAX_MIDDLEWARE_METADATA_BYTES: usize = 32 * 1024;
|
|
|
|
const MAX_MIDDLEWARE_HEADER_MUTATION_WIRE_BYTES: usize = 64 * 1024;
|
|
const MAX_MIDDLEWARE_PROTOBUF_OVERHEAD_BYTES: usize = 64 * 1024;
|
|
const MAX_MIDDLEWARE_REQUEST_ENVELOPE_BYTES: usize = MAX_MIDDLEWARE_CONFIG_BYTES
|
|
+ MAX_MIDDLEWARE_CONTEXT_BYTES
|
|
+ MAX_MIDDLEWARE_TARGET_BYTES
|
|
+ MAX_MIDDLEWARE_HEADER_BYTES
|
|
+ MAX_MIDDLEWARE_PROTOBUF_OVERHEAD_BYTES;
|
|
const MAX_MIDDLEWARE_RESPONSE_ENVELOPE_BYTES: usize = MAX_MIDDLEWARE_REASON_BYTES
|
|
+ MAX_MIDDLEWARE_REASON_CODE_BYTES
|
|
+ MAX_MIDDLEWARE_HEADER_MUTATION_WIRE_BYTES
|
|
+ MAX_MIDDLEWARE_FINDINGS_PER_STAGE * MAX_MIDDLEWARE_FINDING_BYTES
|
|
+ MAX_MIDDLEWARE_METADATA_BYTES
|
|
+ MAX_MIDDLEWARE_PROTOBUF_OVERHEAD_BYTES;
|
|
/// gRPC envelope headroom derived from every bounded non-payload component.
|
|
pub const MIDDLEWARE_GRPC_ENVELOPE_BYTES: usize =
|
|
if MAX_MIDDLEWARE_REQUEST_ENVELOPE_BYTES > MAX_MIDDLEWARE_RESPONSE_ENVELOPE_BYTES {
|
|
MAX_MIDDLEWARE_REQUEST_ENVELOPE_BYTES
|
|
} else {
|
|
MAX_MIDDLEWARE_RESPONSE_ENVELOPE_BYTES
|
|
};
|
|
/// gRPC message limit derived from the payload and bounded protobuf components.
|
|
pub const MIDDLEWARE_GRPC_MESSAGE_BYTES: usize =
|
|
MAX_MIDDLEWARE_PAYLOAD_BYTES + MIDDLEWARE_GRPC_ENVELOPE_BYTES;
|
|
|
|
const MAX_STABLE_IDENTIFIER_BYTES: usize = 128;
|
|
const EXTERNAL_FINDING_LABEL: &str = "External middleware finding";
|
|
#[cfg(test)]
|
|
const HTTP_REQUEST_OPERATION: SupervisorMiddlewareOperation =
|
|
SupervisorMiddlewareOperation::HttpRequest;
|
|
const PRE_CREDENTIALS_PHASE: SupervisorMiddlewarePhase = SupervisorMiddlewarePhase::PreCredentials;
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
pub enum OnError {
|
|
FailClosed,
|
|
FailOpen,
|
|
}
|
|
|
|
impl OnError {
|
|
pub fn parse(value: &str) -> Result<Self> {
|
|
match value {
|
|
"" | "fail_closed" => Ok(Self::FailClosed),
|
|
"fail_open" => Ok(Self::FailOpen),
|
|
other => Err(miette!(
|
|
"invalid middleware on_error '{other}', expected fail_closed or fail_open"
|
|
)),
|
|
}
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ChainEntry {
|
|
pub name: String,
|
|
pub implementation: String,
|
|
pub order: i32,
|
|
pub config: prost_types::Struct,
|
|
pub on_error: OnError,
|
|
}
|
|
|
|
impl TryFrom<(&str, &NetworkMiddlewareConfig)> for ChainEntry {
|
|
type Error = miette::Report;
|
|
|
|
fn try_from((name, value): (&str, &NetworkMiddlewareConfig)) -> Result<Self> {
|
|
if name.is_empty() {
|
|
return Err(miette!("middleware config name cannot be empty"));
|
|
}
|
|
if value.middleware.is_empty() {
|
|
return Err(miette!(
|
|
"middleware config '{}' must reference a middleware",
|
|
name
|
|
));
|
|
}
|
|
Ok(Self {
|
|
name: name.to_string(),
|
|
implementation: value.middleware.clone(),
|
|
order: value.order,
|
|
config: value.config.clone().unwrap_or_default(),
|
|
on_error: OnError::parse(&value.on_error)?,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// A policy-selected middleware config joined with metadata reported by its
|
|
/// service's `Describe` call.
|
|
///
|
|
/// An unregistered implementation is retained so `on_error` can decide whether
|
|
/// the request fails open or closed. A registered implementation without the
|
|
/// requested binding is not part of this chain.
|
|
#[derive(Clone)]
|
|
pub struct DescribedChainEntry {
|
|
entry: ChainEntry,
|
|
service: Option<Arc<MiddlewareServiceState>>,
|
|
binding: Option<MiddlewareBinding>,
|
|
max_payload_bytes: usize,
|
|
timeout: Duration,
|
|
}
|
|
|
|
struct DescribedChain {
|
|
entries: Vec<DescribedChainEntry>,
|
|
unbound: Vec<ChainEntry>,
|
|
}
|
|
|
|
impl DescribedChainEntry {
|
|
pub fn max_payload_bytes(&self) -> usize {
|
|
self.max_payload_bytes
|
|
}
|
|
|
|
pub fn on_error(&self) -> OnError {
|
|
self.entry.on_error
|
|
}
|
|
|
|
pub fn timeout(&self) -> Duration {
|
|
self.timeout
|
|
}
|
|
|
|
/// True when this entry resolved to a registered binding and will be
|
|
/// evaluated. When false, the binding is absent from the current registry
|
|
/// and the entry is handled entirely by its `on_error` policy, so it
|
|
/// imposes no payload-buffering limit on the chain.
|
|
pub fn is_resolved(&self) -> bool {
|
|
self.binding.is_some()
|
|
}
|
|
}
|
|
|
|
/// Re-checks a middleware-transformed request body against sandbox policy.
|
|
///
|
|
/// Returns `Some(reason)` to deny the chain, `None` to proceed. Invoked after
|
|
/// each stage that replaces the body so neither a later stage nor the upstream
|
|
/// sees a payload the policy would reject. Protocols with no body-aware policy
|
|
/// select [`TransformedBodyPolicy::NotPolicyRelevant`] instead.
|
|
pub type TransformedBodyValidator<'a> = dyn Fn(&[u8]) -> Result<Option<String>> + Send + Sync + 'a;
|
|
|
|
/// Whether middleware body replacements affect the selected request policy.
|
|
///
|
|
/// The network pipeline must choose a mode explicitly. This avoids representing
|
|
/// a security-relevant re-evaluation requirement as an optional callback where
|
|
/// an omitted value is indistinguishable from an intentionally body-independent
|
|
/// protocol.
|
|
#[derive(Clone, Copy)]
|
|
pub enum TransformedBodyPolicy<'a> {
|
|
/// The selected policy does not inspect the request body.
|
|
NotPolicyRelevant,
|
|
/// Re-evaluate every body replacement before the next stage runs.
|
|
Reevaluate(&'a TransformedBodyValidator<'a>),
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct HttpRequestInput {
|
|
pub request_id: String,
|
|
pub sandbox_id: String,
|
|
pub sandbox_name: String,
|
|
pub workspace: String,
|
|
pub scheme: String,
|
|
pub host: String,
|
|
pub port: u16,
|
|
pub method: String,
|
|
pub path: String,
|
|
pub query: String,
|
|
/// Lowercased request headers in wire order. Repeated header names are
|
|
/// preserved as separate entries so middleware inspects every value the
|
|
/// upstream will receive.
|
|
pub headers: Vec<(String, String)>,
|
|
/// Lowercased names nominated by the original request's `Connection`
|
|
/// headers. Their values are not exposed to middleware, but mutations must
|
|
/// still treat these dynamically hop-by-hop fields as protected.
|
|
pub connection_nominated_headers: Vec<String>,
|
|
pub body: Vec<u8>,
|
|
}
|
|
|
|
#[derive(Debug, Clone)]
|
|
pub struct ChainOutcome {
|
|
pub allowed: bool,
|
|
pub reason: String,
|
|
pub body: Vec<u8>,
|
|
/// Ordered, validated mutations to replay against the original raw request.
|
|
pub header_mutations: Vec<HeaderMutation>,
|
|
pub findings: Vec<NamespacedFinding>,
|
|
pub metadata: BTreeMap<String, BTreeMap<String, String>>,
|
|
pub applied: Vec<MiddlewareInvocation>,
|
|
/// Present only when a middleware completed successfully and explicitly
|
|
/// denied the request. Fail-closed service errors and transformed-body
|
|
/// policy denials are not represented as middleware decisions.
|
|
pub denial: Option<MiddlewareDenial>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct MiddlewareDenial {
|
|
/// Stable policy-local middleware config identity.
|
|
pub config_name: String,
|
|
/// Validated service-defined code. Free-form service reason text is never
|
|
/// carried into client responses or security logs.
|
|
pub reason_code: Option<String>,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct NamespacedFinding {
|
|
pub middleware: String,
|
|
pub finding: Finding,
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct MiddlewareInvocation {
|
|
pub name: String,
|
|
pub implementation: String,
|
|
pub decision: Decision,
|
|
pub transformed: bool,
|
|
/// True when the middleware could not be evaluated and `on_error` was applied
|
|
/// (service error, malformed/unsafe response, etc.). The `decision` reflects
|
|
/// the `on_error` outcome, not a decision the middleware actually returned.
|
|
pub failed: bool,
|
|
}
|
|
|
|
enum OnErrorAction {
|
|
/// `fail_open`: skip this middleware, leaving the request unchanged.
|
|
FailOpen,
|
|
/// `fail_closed`: short-circuit the chain and deny with the given reason.
|
|
FailClosed(String),
|
|
}
|
|
|
|
/// Apply a middleware entry's `on_error` policy after a failure (service error or
|
|
/// malformed response). Records a `failed` invocation for telemetry in both cases.
|
|
fn apply_on_error(
|
|
entry: &DescribedChainEntry,
|
|
reason: &str,
|
|
applied: &mut Vec<MiddlewareInvocation>,
|
|
) -> OnErrorAction {
|
|
match entry.entry.on_error {
|
|
OnError::FailOpen => {
|
|
applied.push(MiddlewareInvocation {
|
|
name: entry.entry.name.clone(),
|
|
implementation: entry.entry.implementation.clone(),
|
|
decision: Decision::Allow,
|
|
transformed: false,
|
|
failed: true,
|
|
});
|
|
OnErrorAction::FailOpen
|
|
}
|
|
OnError::FailClosed => {
|
|
applied.push(MiddlewareInvocation {
|
|
name: entry.entry.name.clone(),
|
|
implementation: entry.entry.implementation.clone(),
|
|
decision: Decision::Deny,
|
|
transformed: false,
|
|
failed: true,
|
|
});
|
|
OnErrorAction::FailClosed(format!("middleware_failed: {reason}"))
|
|
}
|
|
}
|
|
}
|
|
|
|
fn request_view_to_evaluation(request: HttpRequestView<'_>) -> HttpRequestEvaluation {
|
|
HttpRequestEvaluation {
|
|
phase: request.phase() as i32,
|
|
context: Some(request.context().clone()),
|
|
config: Some(request.config().clone()),
|
|
target: Some(request.target().clone()),
|
|
headers: request.headers().to_vec(),
|
|
body: request.body().to_vec(),
|
|
middleware_name: request.middleware_name().to_string(),
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
pub struct ChainRunner {
|
|
registry: Arc<MiddlewareRegistry>,
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
enum MiddlewareDispatch {
|
|
/// Built-ins borrow the current request state and never construct protobuf.
|
|
InProcess(Arc<dyn InProcessMiddleware>),
|
|
/// Operator services receive an owned protobuf through the gRPC adapter.
|
|
Grpc(remote::GrpcMiddlewareService),
|
|
}
|
|
|
|
impl MiddlewareDispatch {
|
|
async fn describe(
|
|
&self,
|
|
) -> std::result::Result<tonic::Response<MiddlewareManifest>, tonic::Status> {
|
|
match self {
|
|
Self::InProcess(service) => Ok(tonic::Response::new(service.describe().await)),
|
|
Self::Grpc(service) => service.describe().await,
|
|
}
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
middleware_name: &str,
|
|
config: &prost_types::Struct,
|
|
) -> std::result::Result<tonic::Response<ValidateConfigResponse>, tonic::Status> {
|
|
match self {
|
|
Self::InProcess(service) => Ok(tonic::Response::new(
|
|
match service.validate_config(middleware_name, config).await {
|
|
Ok(()) => ValidateConfigResponse {
|
|
valid: true,
|
|
reason: String::new(),
|
|
},
|
|
Err(error) => ValidateConfigResponse {
|
|
valid: false,
|
|
reason: error.to_string(),
|
|
},
|
|
},
|
|
)),
|
|
Self::Grpc(service) => service.validate_config(middleware_name, config).await,
|
|
}
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: HttpRequestView<'_>,
|
|
) -> std::result::Result<tonic::Response<openshell_core::proto::HttpRequestResult>, tonic::Status>
|
|
{
|
|
match self {
|
|
Self::InProcess(service) => service
|
|
.evaluate_http_request(request)
|
|
.await
|
|
.map(tonic::Response::new)
|
|
.map_err(|error| tonic::Status::invalid_argument(error.to_string())),
|
|
Self::Grpc(service) => service.evaluate_http_request(request).await,
|
|
}
|
|
}
|
|
|
|
async fn open_websocket_session(
|
|
&self,
|
|
receiver: tokio::sync::mpsc::Receiver<openshell_core::proto::WebSocketSessionEvent>,
|
|
) -> std::result::Result<WebSocketResponseStream, tonic::Status> {
|
|
match self {
|
|
Self::InProcess(service) => service.open_websocket_session(receiver).await,
|
|
Self::Grpc(service) => service.open_websocket_session(receiver).await,
|
|
}
|
|
}
|
|
|
|
async fn open_http_response_pre_return(
|
|
&self,
|
|
receiver: tokio::sync::mpsc::Receiver<openshell_core::proto::HttpResponseEvent>,
|
|
) -> std::result::Result<HttpResponseResultStream, tonic::Status> {
|
|
match self {
|
|
Self::InProcess(service) => service.open_http_response_pre_return(receiver).await,
|
|
Self::Grpc(service) => service.open_http_response_pre_return(receiver).await,
|
|
}
|
|
}
|
|
}
|
|
|
|
struct MiddlewareServiceState {
|
|
/// Policy-facing built-in name or operator-owned registration name. The
|
|
/// single-service test constructor leaves this empty and uses the manifest
|
|
/// name after Describe.
|
|
attachment_name: Option<String>,
|
|
service: MiddlewareDispatch,
|
|
manifest: OnceCell<MiddlewareManifest>,
|
|
diagnostic_policy: MiddlewareDiagnosticPolicy,
|
|
operator_max_payload_bytes: Option<usize>,
|
|
operator_timeout: Duration,
|
|
}
|
|
|
|
impl MiddlewareServiceState {
|
|
fn timeout_for_binding(&self, binding: &MiddlewareBinding) -> Result<Duration> {
|
|
if binding.request_timeout.is_none() {
|
|
Ok(self.operator_timeout)
|
|
} else {
|
|
middleware_proto_timeout_or_default(binding.request_timeout.as_ref())
|
|
.map(|binding_timeout| binding_timeout.min(self.operator_timeout))
|
|
.map_err(|reason| miette!("middleware binding has invalid timeout: {reason}"))
|
|
}
|
|
}
|
|
}
|
|
|
|
async fn call_with_timeout<T>(
|
|
timeout: Duration,
|
|
operation: &'static str,
|
|
future: impl Future<Output = std::result::Result<tonic::Response<T>, tonic::Status>>,
|
|
) -> std::result::Result<tonic::Response<T>, tonic::Status> {
|
|
tokio::time::timeout(timeout, future).await.map_err(|_| {
|
|
tonic::Status::deadline_exceeded(format!("middleware {operation} timed out"))
|
|
})?
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum MiddlewareDiagnosticPolicy {
|
|
Preserve,
|
|
Normalize,
|
|
}
|
|
|
|
impl MiddlewareDiagnosticPolicy {
|
|
fn error_reason(self, error: &tonic::Status) -> String {
|
|
match self {
|
|
Self::Preserve => safe_reason(&error.to_string()),
|
|
Self::Normalize => "external_service_error".to_string(),
|
|
}
|
|
}
|
|
|
|
fn process_result(
|
|
self,
|
|
middleware_name: &str,
|
|
result: &mut openshell_core::proto::HttpRequestResult,
|
|
) {
|
|
if self == Self::Normalize {
|
|
normalize_untrusted_diagnostics(middleware_name, result);
|
|
}
|
|
}
|
|
|
|
fn header_mutation_error_reason(self, error: &headers::HeaderMutationError) -> String {
|
|
match self {
|
|
Self::Preserve => safe_reason(&error.to_string()),
|
|
Self::Normalize => error.code().to_string(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Validated middleware services available to a gateway or one supervisor.
|
|
///
|
|
/// In-process services are supplied by the composition root; the generic
|
|
/// registry does not select concrete built-ins. All in-process and remote
|
|
/// services are described before construction succeeds, so callers never
|
|
/// observe a partially registered service set.
|
|
#[derive(Clone)]
|
|
pub struct MiddlewareRegistry {
|
|
services: Arc<Vec<Arc<MiddlewareServiceState>>>,
|
|
registered_services: Arc<Vec<RegisteredMiddlewareService>>,
|
|
middleware_names: Arc<HashSet<String>>,
|
|
negotiated_extensions: Arc<Vec<NegotiatedExtension>>,
|
|
work_admission: Arc<Semaphore>,
|
|
work_admission_waiters: Arc<Semaphore>,
|
|
session_admission: Arc<Semaphore>,
|
|
}
|
|
|
|
impl std::fmt::Debug for MiddlewareRegistry {
|
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
|
formatter
|
|
.debug_struct("MiddlewareRegistry")
|
|
.field("service_count", &self.services.len())
|
|
.field("registered_service_count", &self.registered_services.len())
|
|
.field("middleware_count", &self.middleware_names.len())
|
|
.field(
|
|
"available_work_permits",
|
|
&self.work_admission.available_permits(),
|
|
)
|
|
.field(
|
|
"available_session_permits",
|
|
&self.session_admission.available_permits(),
|
|
)
|
|
.finish()
|
|
}
|
|
}
|
|
|
|
#[derive(Clone)]
|
|
struct RegisteredMiddlewareService {
|
|
registration: SupervisorMiddlewareService,
|
|
}
|
|
|
|
impl Default for MiddlewareRegistry {
|
|
fn default() -> Self {
|
|
Self {
|
|
services: Arc::new(Vec::new()),
|
|
registered_services: Arc::new(Vec::new()),
|
|
middleware_names: Arc::new(HashSet::new()),
|
|
negotiated_extensions: Arc::new(Vec::new()),
|
|
work_admission: Arc::new(Semaphore::new(MAX_CONCURRENT_MIDDLEWARE_WORK)),
|
|
work_admission_waiters: Arc::new(Semaphore::new(MAX_QUEUED_MIDDLEWARE_WORK)),
|
|
session_admission: Arc::new(Semaphore::new(MAX_CONCURRENT_MIDDLEWARE_SESSIONS)),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Validate one external middleware registration without opening its transport.
|
|
pub fn validate_registration_config(registration: &SupervisorMiddlewareService) -> Result<()> {
|
|
validate_registration(registration).map(|_| ())
|
|
}
|
|
|
|
fn validate_registration(registration: &SupervisorMiddlewareService) -> Result<Duration> {
|
|
if !is_stable_identifier(®istration.name) {
|
|
return Err(miette!(
|
|
"supervisor middleware registration names must be 1-{MAX_STABLE_IDENTIFIER_BYTES} bytes and contain only ASCII letters, digits, '.', '_', '-', or '/'"
|
|
));
|
|
}
|
|
if registration.name.starts_with("openshell/") {
|
|
return Err(miette!(
|
|
"middleware registration '{}' cannot claim the reserved openshell/ namespace",
|
|
registration.name
|
|
));
|
|
}
|
|
if !registration.grpc_endpoint.starts_with("http://")
|
|
&& !registration.grpc_endpoint.starts_with("https://")
|
|
{
|
|
return Err(miette!(
|
|
"middleware registration '{}' grpc_endpoint must use http:// or https://",
|
|
registration.name
|
|
));
|
|
}
|
|
if registration.max_payload_bytes > MAX_MIDDLEWARE_PAYLOAD_BYTES as u64 {
|
|
return Err(miette!(
|
|
"middleware registration '{}' max_payload_bytes exceeds the platform maximum of {MAX_MIDDLEWARE_PAYLOAD_BYTES}",
|
|
registration.name
|
|
));
|
|
}
|
|
middleware_proto_timeout_or_default(registration.request_timeout.as_ref()).map_err(|reason| {
|
|
miette!(
|
|
"middleware registration '{}' has invalid timeout: {reason}",
|
|
registration.name
|
|
)
|
|
})
|
|
}
|
|
|
|
fn is_stable_identifier(value: &str) -> bool {
|
|
!value.is_empty()
|
|
&& value.len() <= MAX_STABLE_IDENTIFIER_BYTES
|
|
&& value
|
|
.bytes()
|
|
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'-' | b'/'))
|
|
}
|
|
|
|
fn is_stable_reason_code(value: &str) -> bool {
|
|
value.len() <= MAX_MIDDLEWARE_REASON_CODE_BYTES
|
|
&& value.as_bytes().first().is_some_and(u8::is_ascii_lowercase)
|
|
&& value
|
|
.bytes()
|
|
.all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_')
|
|
}
|
|
|
|
fn middleware_denial_reason(config_name: &str, reason_code: Option<&str>) -> String {
|
|
let config_id: String = config_name
|
|
.chars()
|
|
.map(|character| {
|
|
if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') {
|
|
character
|
|
} else {
|
|
'_'
|
|
}
|
|
})
|
|
.take(MAX_STABLE_IDENTIFIER_BYTES)
|
|
.collect();
|
|
reason_code.map_or_else(
|
|
|| format!("middleware_denied:{config_id}"),
|
|
|code| format!("middleware_denied:{config_id}:{code}"),
|
|
)
|
|
}
|
|
|
|
fn validate_payload_limit(source: &str, binding: &MiddlewareBinding) -> Result<usize> {
|
|
if binding.max_payload_bytes == 0 {
|
|
return Err(miette!("{source} must advertise a non-zero payload limit"));
|
|
}
|
|
if binding.max_payload_bytes > MAX_MIDDLEWARE_PAYLOAD_BYTES as u64 {
|
|
return Err(miette!(
|
|
"{source} payload limit exceeds the platform maximum of {MAX_MIDDLEWARE_PAYLOAD_BYTES}"
|
|
));
|
|
}
|
|
usize::try_from(binding.max_payload_bytes)
|
|
.map_err(|_| miette!("{source} reports a payload limit too large for this platform"))
|
|
}
|
|
|
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
enum SupportedBinding {
|
|
HttpPreCredentials,
|
|
HttpResponsePreReturn,
|
|
WebSocketPreCredentials,
|
|
}
|
|
|
|
fn supported_binding(source: &str, binding: &MiddlewareBinding) -> Result<SupportedBinding> {
|
|
match (
|
|
SupervisorMiddlewareOperation::try_from(binding.operation).ok(),
|
|
SupervisorMiddlewarePhase::try_from(binding.phase).ok(),
|
|
) {
|
|
(
|
|
Some(SupervisorMiddlewareOperation::HttpRequest),
|
|
Some(SupervisorMiddlewarePhase::PreCredentials),
|
|
) => Ok(SupportedBinding::HttpPreCredentials),
|
|
(
|
|
Some(SupervisorMiddlewareOperation::HttpResponse),
|
|
Some(SupervisorMiddlewarePhase::PreReturn),
|
|
) => Ok(SupportedBinding::HttpResponsePreReturn),
|
|
(
|
|
Some(SupervisorMiddlewareOperation::WebsocketMessage),
|
|
Some(SupervisorMiddlewarePhase::PreCredentials),
|
|
) => Ok(SupportedBinding::WebSocketPreCredentials),
|
|
(
|
|
Some(SupervisorMiddlewareOperation::WebsocketMessage),
|
|
Some(SupervisorMiddlewarePhase::PreReturn),
|
|
) => Err(miette!(
|
|
"{source} advertises WEBSOCKET_MESSAGE/PRE_RETURN, which is not yet supported"
|
|
)),
|
|
_ => Err(miette!(
|
|
"{source} advertises an unsupported middleware operation/phase pair"
|
|
)),
|
|
}
|
|
}
|
|
|
|
fn validate_manifest_bindings(
|
|
source: &str,
|
|
manifest: &MiddlewareManifest,
|
|
operator_max_payload_bytes: Option<usize>,
|
|
) -> Result<()> {
|
|
if manifest.bindings.is_empty() {
|
|
return Err(miette!("{source} describes no bindings"));
|
|
}
|
|
|
|
let mut described_pairs = HashSet::with_capacity(manifest.bindings.len());
|
|
for binding in &manifest.bindings {
|
|
supported_binding(source, binding)?;
|
|
if !described_pairs.insert((binding.operation, binding.phase)) {
|
|
return Err(miette!(
|
|
"{source} describes a duplicate middleware operation/phase pair"
|
|
));
|
|
}
|
|
let advertised = validate_payload_limit(source, binding)?;
|
|
if binding.request_timeout.is_some() {
|
|
middleware_proto_timeout_or_default(binding.request_timeout.as_ref())
|
|
.map_err(|reason| miette!("{source} has invalid timeout for binding: {reason}"))?;
|
|
}
|
|
if operator_max_payload_bytes.is_some_and(|limit| limit > advertised) {
|
|
return Err(miette!(
|
|
"{source} max_payload_bytes ({}) exceeds the binding capability ({advertised})",
|
|
operator_max_payload_bytes.expect("operator limit checked above")
|
|
));
|
|
}
|
|
if operator_max_payload_bytes == Some(0) {
|
|
return Err(miette!(
|
|
"{source} must configure max_payload_bytes for every payload-bearing binding"
|
|
));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn middleware_proto_timeout_or_default(
|
|
value: Option<&prost_types::Duration>,
|
|
) -> std::result::Result<Duration, String> {
|
|
let Some(value) = value else {
|
|
return Ok(DEFAULT_MIDDLEWARE_TIMEOUT);
|
|
};
|
|
let timeout =
|
|
openshell_core::time::duration_to_std(value).map_err(|error| error.to_string())?;
|
|
if !(MIN_MIDDLEWARE_TIMEOUT..=MAX_MIDDLEWARE_TIMEOUT).contains(&timeout) {
|
|
return Err(format!(
|
|
"must be between {}ms and {}s",
|
|
MIN_MIDDLEWARE_TIMEOUT.as_millis(),
|
|
MAX_MIDDLEWARE_TIMEOUT.as_secs()
|
|
));
|
|
}
|
|
Ok(timeout)
|
|
}
|
|
|
|
fn validate_external_manifest(
|
|
registration: &SupervisorMiddlewareService,
|
|
manifest: &MiddlewareManifest,
|
|
operator_max_payload_bytes: usize,
|
|
authenticated: bool,
|
|
) -> Result<()> {
|
|
validate_manifest_bindings(
|
|
&format!("external middleware registration '{}'", registration.name),
|
|
manifest,
|
|
Some(operator_max_payload_bytes),
|
|
)?;
|
|
validate_expected_audience(
|
|
®istration.name,
|
|
®istration.audience,
|
|
&manifest.expected_audience,
|
|
authenticated && !registration.allow_insecure_transport,
|
|
)
|
|
}
|
|
|
|
/// After authenticated Describe succeeds, reject a registration whose
|
|
/// configured audience differs from the one the service says it verifies.
|
|
///
|
|
/// This is a post-authentication consistency assertion, not audience discovery:
|
|
/// a strict verifier may reject an incorrect audience before returning its
|
|
/// manifest. A service that does not advertise an audience is accepted unchanged.
|
|
fn validate_expected_audience(
|
|
registration_name: &str,
|
|
configured: &str,
|
|
advertised: &str,
|
|
authenticated: bool,
|
|
) -> Result<()> {
|
|
if !authenticated || advertised.is_empty() {
|
|
return Ok(());
|
|
}
|
|
if advertised != configured {
|
|
return Err(miette!(
|
|
"middleware registration '{registration_name}' expects audience \
|
|
'{advertised}' but OpenShell is configured to mint '{configured}'"
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// External diagnostic text is untrusted and may contain request data. Keep
|
|
/// only values derived from the validated, operator-owned registration name
|
|
/// and numeric finding counts; do not carry per-request free-form text into
|
|
/// logs.
|
|
fn normalize_untrusted_diagnostics(
|
|
middleware_name: &str,
|
|
result: &mut openshell_core::proto::HttpRequestResult,
|
|
) {
|
|
let reason_id: String = middleware_name
|
|
.chars()
|
|
.map(|character| {
|
|
if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') {
|
|
character
|
|
} else {
|
|
'_'
|
|
}
|
|
})
|
|
.collect();
|
|
result.reason = format!("middleware_denied:{reason_id}");
|
|
result.metadata.clear();
|
|
for finding in &mut result.findings {
|
|
finding.r#type = format!("{middleware_name}.finding");
|
|
finding.label = EXTERNAL_FINDING_LABEL.to_string();
|
|
finding.confidence.clear();
|
|
finding.severity = match finding.severity.as_str() {
|
|
"low" => "low",
|
|
"high" => "high",
|
|
_ => "medium",
|
|
}
|
|
.to_string();
|
|
}
|
|
}
|
|
|
|
fn validate_request_view(request: HttpRequestView<'_>) -> std::result::Result<(), &'static str> {
|
|
if request.body().len() > MAX_MIDDLEWARE_PAYLOAD_BYTES {
|
|
return Err("request_body_over_capacity");
|
|
}
|
|
if request.config().encoded_len() > MAX_MIDDLEWARE_CONFIG_BYTES {
|
|
return Err("request_config_over_capacity");
|
|
}
|
|
if request.context().encoded_len() > MAX_MIDDLEWARE_CONTEXT_BYTES {
|
|
return Err("request_context_over_capacity");
|
|
}
|
|
if request.target().encoded_len() > MAX_MIDDLEWARE_TARGET_BYTES {
|
|
return Err("request_target_over_capacity");
|
|
}
|
|
if request.headers().len() > MAX_MIDDLEWARE_HEADERS {
|
|
return Err("request_header_count_over_capacity");
|
|
}
|
|
let header_bytes = request.headers().iter().fold(0usize, |total, header| {
|
|
total.saturating_add(header.encoded_len())
|
|
});
|
|
if header_bytes > MAX_MIDDLEWARE_HEADER_BYTES {
|
|
return Err("request_header_bytes_over_capacity");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn validate_response_envelope(
|
|
result: &openshell_core::proto::HttpRequestResult,
|
|
) -> std::result::Result<(), &'static str> {
|
|
if result.body.len() > MAX_MIDDLEWARE_PAYLOAD_BYTES {
|
|
return Err("response_body_over_capacity");
|
|
}
|
|
if result.reason.len() > MAX_MIDDLEWARE_REASON_BYTES {
|
|
return Err("response_reason_over_capacity");
|
|
}
|
|
if !result.reason_code.is_empty() && !is_stable_reason_code(&result.reason_code) {
|
|
return Err("response_reason_code_invalid");
|
|
}
|
|
if result.header_mutations.len() > headers::MAX_HEADER_MUTATIONS {
|
|
return Err("header_mutation_count_over_capacity");
|
|
}
|
|
let mutation_bytes = result
|
|
.header_mutations
|
|
.iter()
|
|
.fold(0usize, |total, mutation| {
|
|
total.saturating_add(mutation.encoded_len())
|
|
});
|
|
if mutation_bytes > MAX_MIDDLEWARE_HEADER_MUTATION_WIRE_BYTES {
|
|
return Err("header_mutation_bytes_over_capacity");
|
|
}
|
|
if result.findings.len() > MAX_MIDDLEWARE_FINDINGS_PER_STAGE {
|
|
return Err("response_findings_over_capacity");
|
|
}
|
|
if result
|
|
.findings
|
|
.iter()
|
|
.any(|finding| finding.encoded_len() > MAX_MIDDLEWARE_FINDING_BYTES)
|
|
{
|
|
return Err("response_finding_over_capacity");
|
|
}
|
|
if result.metadata.len() > MAX_MIDDLEWARE_METADATA_ENTRIES {
|
|
return Err("response_metadata_count_over_capacity");
|
|
}
|
|
let metadata_bytes = result.metadata.iter().fold(0usize, |total, (key, value)| {
|
|
total.saturating_add(key.len()).saturating_add(value.len())
|
|
});
|
|
if metadata_bytes > MAX_MIDDLEWARE_METADATA_BYTES {
|
|
return Err("response_metadata_bytes_over_capacity");
|
|
}
|
|
if result.encoded_len() > MIDDLEWARE_GRPC_MESSAGE_BYTES {
|
|
return Err("response_envelope_over_capacity");
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
impl MiddlewareRegistry {
|
|
/// Describe in-process services, then connect and validate every
|
|
/// operator-provided service registration.
|
|
pub async fn connect_services(
|
|
in_process_services: Vec<Arc<dyn InProcessMiddleware>>,
|
|
registrations: Vec<SupervisorMiddlewareService>,
|
|
) -> Result<Self> {
|
|
Self::connect_services_inner(in_process_services, registrations, None).await
|
|
}
|
|
|
|
/// Connect services with optional refreshable credentials keyed by
|
|
/// operator registration name. A configured credential is shared by all
|
|
/// generated client clones and can rotate without rebuilding the registry.
|
|
pub async fn connect_services_authenticated(
|
|
in_process_services: Vec<Arc<dyn InProcessMiddleware>>,
|
|
registrations: Vec<SupervisorMiddlewareService>,
|
|
credentials: &HashMap<String, openshell_extension_core::BearerTokenSlot>,
|
|
) -> Result<Self> {
|
|
Self::connect_services_inner(in_process_services, registrations, Some(credentials)).await
|
|
}
|
|
|
|
async fn connect_services_inner(
|
|
in_process_services: Vec<Arc<dyn InProcessMiddleware>>,
|
|
registrations: Vec<SupervisorMiddlewareService>,
|
|
credentials: Option<&HashMap<String, openshell_extension_core::BearerTokenSlot>>,
|
|
) -> Result<Self> {
|
|
let mut services = Vec::with_capacity(in_process_services.len() + registrations.len());
|
|
let mut registered_services = Vec::with_capacity(registrations.len());
|
|
let mut middleware_names = HashSet::new();
|
|
let mut negotiated_extensions = Vec::new();
|
|
let gateway = gateway_metadata(ExtensionFamily::SupervisorMiddleware);
|
|
|
|
for service in in_process_services {
|
|
let service = MiddlewareDispatch::InProcess(service);
|
|
let manifest =
|
|
call_with_timeout(DEFAULT_MIDDLEWARE_TIMEOUT, "Describe", service.describe())
|
|
.await
|
|
.map(tonic::Response::into_inner)
|
|
.map_err(|error| {
|
|
miette!(
|
|
"in-process middleware Describe failed: {}",
|
|
safe_reason(&error.to_string())
|
|
)
|
|
})?;
|
|
let source = if manifest.name.trim().is_empty() {
|
|
"in-process middleware service".to_string()
|
|
} else {
|
|
format!("in-process middleware service '{}'", manifest.name)
|
|
};
|
|
if !is_stable_identifier(&manifest.name) {
|
|
return Err(miette!(
|
|
"in-process middleware names must be 1-{MAX_STABLE_IDENTIFIER_BYTES} bytes and contain only ASCII letters, digits, '.', '_', '-', or '/'"
|
|
));
|
|
}
|
|
if !middleware_names.insert(manifest.name.clone()) {
|
|
return Err(miette!(
|
|
"duplicate supervisor middleware name '{}'",
|
|
manifest.name
|
|
));
|
|
}
|
|
validate_manifest_bindings(&source, &manifest, None)?;
|
|
negotiated_extensions.push(
|
|
negotiate(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
&manifest.name,
|
|
&gateway,
|
|
manifest.extension.clone(),
|
|
)
|
|
.map_err(|error| miette!(error.to_string()))?,
|
|
);
|
|
let attachment_name = manifest.name.clone();
|
|
let manifest_cell = OnceCell::new();
|
|
manifest_cell
|
|
.set(manifest)
|
|
.map_err(|_| miette!("middleware manifest cache initialized twice"))?;
|
|
services.push(Arc::new(MiddlewareServiceState {
|
|
attachment_name: Some(attachment_name),
|
|
service,
|
|
manifest: manifest_cell,
|
|
diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve,
|
|
operator_max_payload_bytes: None,
|
|
operator_timeout: DEFAULT_MIDDLEWARE_TIMEOUT,
|
|
}));
|
|
}
|
|
|
|
for registration in registrations {
|
|
let operator_timeout = validate_registration(®istration)?;
|
|
if !middleware_names.insert(registration.name.clone()) {
|
|
return Err(miette!(
|
|
"duplicate supervisor middleware registration name '{}'",
|
|
registration.name
|
|
));
|
|
}
|
|
|
|
let operator_max_payload_bytes = usize::try_from(registration.max_payload_bytes)
|
|
.map_err(|_| {
|
|
miette!(
|
|
"middleware registration '{}' payload limit is too large for this platform",
|
|
registration.name
|
|
)
|
|
})?;
|
|
// A registration the operator opted out of extension
|
|
// authentication carries no credential by design. Every other
|
|
// registration must have one, or the connection fails closed
|
|
// rather than silently downgrading to an unauthenticated call.
|
|
let bearer = credentials
|
|
.filter(|_| !registration.allow_insecure_transport)
|
|
.map(|credentials| {
|
|
credentials.get(®istration.name).cloned().ok_or_else(|| {
|
|
miette!(
|
|
"middleware registration '{}' is missing its extension credential",
|
|
registration.name
|
|
)
|
|
})
|
|
})
|
|
.transpose()?;
|
|
let authenticated = bearer.is_some();
|
|
let service = MiddlewareDispatch::Grpc(
|
|
remote::GrpcMiddlewareService::connect(
|
|
®istration.name,
|
|
®istration.grpc_endpoint,
|
|
®istration.tls_ca_cert_pem,
|
|
bearer,
|
|
)
|
|
.await?,
|
|
);
|
|
let manifest = call_with_timeout(operator_timeout, "Describe", service.describe())
|
|
.await
|
|
.map(tonic::Response::into_inner)
|
|
.map_err(|error| {
|
|
miette!(
|
|
"middleware registration '{}' Describe failed: {}",
|
|
registration.name,
|
|
safe_reason(&error.to_string())
|
|
)
|
|
})?;
|
|
validate_external_manifest(
|
|
®istration,
|
|
&manifest,
|
|
operator_max_payload_bytes,
|
|
authenticated,
|
|
)?;
|
|
negotiated_extensions.push(
|
|
negotiate(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
®istration.name,
|
|
&gateway,
|
|
manifest.extension.clone(),
|
|
)
|
|
.map_err(|error| miette!(error.to_string()))?,
|
|
);
|
|
let manifest_cell = OnceCell::new();
|
|
manifest_cell
|
|
.set(manifest)
|
|
.map_err(|_| miette!("middleware manifest cache initialized twice"))?;
|
|
services.push(Arc::new(MiddlewareServiceState {
|
|
attachment_name: Some(registration.name.clone()),
|
|
service,
|
|
manifest: manifest_cell,
|
|
diagnostic_policy: MiddlewareDiagnosticPolicy::Normalize,
|
|
operator_max_payload_bytes: Some(operator_max_payload_bytes),
|
|
operator_timeout,
|
|
}));
|
|
registered_services.push(RegisteredMiddlewareService { registration });
|
|
}
|
|
|
|
Ok(Self {
|
|
services: Arc::new(services),
|
|
registered_services: Arc::new(registered_services),
|
|
middleware_names: Arc::new(middleware_names),
|
|
negotiated_extensions: Arc::new(negotiated_extensions),
|
|
work_admission: Arc::new(Semaphore::new(MAX_CONCURRENT_MIDDLEWARE_WORK)),
|
|
work_admission_waiters: Arc::new(Semaphore::new(MAX_QUEUED_MIDDLEWARE_WORK)),
|
|
session_admission: Arc::new(Semaphore::new(MAX_CONCURRENT_MIDDLEWARE_SESSIONS)),
|
|
})
|
|
}
|
|
|
|
/// Validate implementation-owned configuration for every middleware entry.
|
|
pub async fn validate_policy_configs(&self, policy: &SandboxPolicy) -> Result<()> {
|
|
ensure_config_capacity(policy.network_middlewares.len())?;
|
|
let runner = ChainRunner::from_registry(self.clone());
|
|
for (name, config) in &policy.network_middlewares {
|
|
runner
|
|
.validate_config(
|
|
&config.middleware,
|
|
config.config.clone().unwrap_or_default(),
|
|
)
|
|
.await
|
|
.map_err(|error| {
|
|
miette!(
|
|
"middleware config '{}' is invalid: {}",
|
|
name,
|
|
safe_reason(&error.to_string())
|
|
)
|
|
})?;
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Check that every policy attachment still belongs to the current static
|
|
/// registry without making a network call.
|
|
pub fn ensure_policy_middlewares_registered(&self, policy: &SandboxPolicy) -> Result<()> {
|
|
for (name, config) in &policy.network_middlewares {
|
|
if !self.middleware_names.contains(&config.middleware) {
|
|
return Err(miette!(
|
|
"middleware '{}' used by config '{}' is not registered",
|
|
config.middleware,
|
|
name
|
|
));
|
|
}
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
/// Return only operator-registered services referenced by the effective policy.
|
|
pub fn required_services(
|
|
&self,
|
|
policy: Option<&SandboxPolicy>,
|
|
) -> Vec<SupervisorMiddlewareService> {
|
|
let Some(policy) = policy else {
|
|
return Vec::new();
|
|
};
|
|
let selected: HashSet<&str> = policy
|
|
.network_middlewares
|
|
.values()
|
|
.map(|config| config.middleware.as_str())
|
|
.collect();
|
|
self.registered_services
|
|
.iter()
|
|
.filter(|service| selected.contains(service.registration.name.as_str()))
|
|
.map(|service| service.registration.clone())
|
|
.collect()
|
|
}
|
|
|
|
#[must_use]
|
|
pub fn negotiated_extensions(&self) -> &[NegotiatedExtension] {
|
|
&self.negotiated_extensions
|
|
}
|
|
}
|
|
|
|
impl Default for ChainRunner {
|
|
fn default() -> Self {
|
|
Self::from_registry(MiddlewareRegistry::default())
|
|
}
|
|
}
|
|
|
|
impl ChainRunner {
|
|
/// Construct a runner around one in-process middleware implementation.
|
|
#[must_use]
|
|
pub fn new(service: Arc<dyn InProcessMiddleware>) -> Self {
|
|
Self::from_service(MiddlewareDispatch::InProcess(service))
|
|
}
|
|
|
|
/// Construct a runner around a legacy transport-neutral in-process endpoint.
|
|
#[must_use]
|
|
pub fn from_endpoint(endpoint: Arc<dyn SupervisorMiddlewareEndpoint>) -> Self {
|
|
Self::new(in_process_endpoint(endpoint))
|
|
}
|
|
|
|
fn from_service(service: MiddlewareDispatch) -> Self {
|
|
Self {
|
|
registry: Arc::new(MiddlewareRegistry {
|
|
services: Arc::new(vec![Arc::new(MiddlewareServiceState {
|
|
attachment_name: None,
|
|
service,
|
|
manifest: OnceCell::new(),
|
|
diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve,
|
|
operator_max_payload_bytes: None,
|
|
operator_timeout: DEFAULT_MIDDLEWARE_TIMEOUT,
|
|
})]),
|
|
registered_services: Arc::new(Vec::new()),
|
|
middleware_names: Arc::new(HashSet::new()),
|
|
negotiated_extensions: Arc::new(Vec::new()),
|
|
work_admission: Arc::new(Semaphore::new(MAX_CONCURRENT_MIDDLEWARE_WORK)),
|
|
work_admission_waiters: Arc::new(Semaphore::new(MAX_QUEUED_MIDDLEWARE_WORK)),
|
|
session_admission: Arc::new(Semaphore::new(MAX_CONCURRENT_MIDDLEWARE_SESSIONS)),
|
|
}),
|
|
}
|
|
}
|
|
|
|
#[cfg(test)]
|
|
fn new_protobuf_for_tests(service: Arc<MiddlewareService>) -> Self {
|
|
let endpoint: Arc<dyn SupervisorMiddlewareEndpoint> =
|
|
Arc::new(GeneratedMiddlewareEndpoint { service });
|
|
Self::from_service(MiddlewareDispatch::Grpc(
|
|
remote::GrpcMiddlewareService::from_service(endpoint),
|
|
))
|
|
}
|
|
|
|
pub fn from_registry(registry: MiddlewareRegistry) -> Self {
|
|
Self {
|
|
registry: Arc::new(registry),
|
|
}
|
|
}
|
|
|
|
/// Build a runner for a replacement registry while preserving process-wide
|
|
/// admission budgets across registry generations.
|
|
#[must_use]
|
|
pub fn with_replacement_registry(&self, mut registry: MiddlewareRegistry) -> Self {
|
|
registry.work_admission = Arc::clone(&self.registry.work_admission);
|
|
registry.work_admission_waiters = Arc::clone(&self.registry.work_admission_waiters);
|
|
registry.session_admission = Arc::clone(&self.registry.session_admission);
|
|
Self::from_registry(registry)
|
|
}
|
|
|
|
/// Reserve one unit of short-lived middleware work.
|
|
///
|
|
/// The bounded waiter queue provides backpressure for work expected to
|
|
/// complete promptly, such as HTTP evaluations, WebSocket messages, and
|
|
/// streaming-session preflight.
|
|
pub async fn reserve_middleware_work(&self) -> Result<MiddlewareWorkAdmissionOutcome> {
|
|
if let Ok(permit) = Arc::clone(&self.registry.work_admission).try_acquire_owned() {
|
|
Ok(MiddlewareWorkAdmissionOutcome::Admitted(
|
|
MiddlewareWorkAdmission {
|
|
_work: permit,
|
|
saturated: false,
|
|
},
|
|
))
|
|
} else {
|
|
let Ok(waiter) = Arc::clone(&self.registry.work_admission_waiters).try_acquire_owned()
|
|
else {
|
|
return Ok(MiddlewareWorkAdmissionOutcome::QueueExhausted);
|
|
};
|
|
let permit = Arc::clone(&self.registry.work_admission)
|
|
.acquire_owned()
|
|
.await
|
|
.map_err(|_| miette!("middleware admission semaphore closed"))?;
|
|
drop(waiter);
|
|
Ok(MiddlewareWorkAdmissionOutcome::Admitted(
|
|
MiddlewareWorkAdmission {
|
|
_work: permit,
|
|
saturated: true,
|
|
},
|
|
))
|
|
}
|
|
}
|
|
|
|
/// Reserve middleware work for a caller whose established external
|
|
/// behavior treats queue exhaustion as a middleware processing failure.
|
|
pub async fn reserve_middleware_work_admission(&self) -> Result<MiddlewareWorkAdmission> {
|
|
self.reserve_middleware_work().await?.into_admission()
|
|
}
|
|
|
|
/// Attempt to reserve one persistent middleware session without waiting.
|
|
///
|
|
/// Long-lived sessions have no useful queueing bound because their release
|
|
/// time is unrelated to middleware latency. Protocol-specific runners apply
|
|
/// their own `on_error` semantics when the shared session budget is full.
|
|
fn try_reserve_middleware_session(&self) -> MiddlewareSessionAdmission {
|
|
Arc::clone(&self.registry.session_admission)
|
|
.try_acquire_owned()
|
|
.map_or(MiddlewareSessionAdmission::AtCapacity, |permit| {
|
|
MiddlewareSessionAdmission::Admitted(MiddlewareSessionPermit { _session: permit })
|
|
})
|
|
}
|
|
|
|
async fn manifests(&self) -> Result<Vec<(Arc<MiddlewareServiceState>, MiddlewareManifest)>> {
|
|
let mut manifests = Vec::with_capacity(self.registry.services.len());
|
|
for state in self.registry.services.iter() {
|
|
let manifest = state
|
|
.manifest
|
|
.get_or_try_init(|| async {
|
|
call_with_timeout(state.operator_timeout, "Describe", state.service.describe())
|
|
.await
|
|
.map(tonic::Response::into_inner)
|
|
.map_err(|error| {
|
|
miette!(
|
|
"middleware Describe failed: {}",
|
|
safe_reason(&error.to_string())
|
|
)
|
|
})
|
|
})
|
|
.await?;
|
|
manifests.push((Arc::clone(state), manifest.clone()));
|
|
}
|
|
Ok(manifests)
|
|
}
|
|
|
|
fn attachment_name<'a>(
|
|
state: &'a MiddlewareServiceState,
|
|
manifest: &'a MiddlewareManifest,
|
|
) -> &'a str {
|
|
state
|
|
.attachment_name
|
|
.as_deref()
|
|
.unwrap_or(manifest.name.as_str())
|
|
}
|
|
|
|
fn binding(
|
|
manifest: &MiddlewareManifest,
|
|
operation: SupervisorMiddlewareOperation,
|
|
phase: SupervisorMiddlewarePhase,
|
|
) -> Option<&MiddlewareBinding> {
|
|
manifest
|
|
.bindings
|
|
.iter()
|
|
.find(|binding| binding.operation == operation as i32 && binding.phase == phase as i32)
|
|
}
|
|
|
|
pub async fn describe_chain(&self, entries: &[ChainEntry]) -> Result<Vec<DescribedChainEntry>> {
|
|
Ok(self
|
|
.describe_chain_for(
|
|
entries,
|
|
SupervisorMiddlewareOperation::HttpRequest,
|
|
SupervisorMiddlewarePhase::PreCredentials,
|
|
)
|
|
.await?
|
|
.entries)
|
|
}
|
|
|
|
pub async fn describe_websocket_chain(
|
|
&self,
|
|
entries: &[ChainEntry],
|
|
) -> Result<Vec<DescribedChainEntry>> {
|
|
Ok(self
|
|
.describe_chain_for(
|
|
entries,
|
|
SupervisorMiddlewareOperation::WebsocketMessage,
|
|
SupervisorMiddlewarePhase::PreCredentials,
|
|
)
|
|
.await?
|
|
.entries)
|
|
}
|
|
|
|
pub async fn describe_http_response_chain(
|
|
&self,
|
|
entries: &[ChainEntry],
|
|
) -> Result<Vec<DescribedChainEntry>> {
|
|
Ok(self
|
|
.describe_chain_for(
|
|
entries,
|
|
SupervisorMiddlewareOperation::HttpResponse,
|
|
SupervisorMiddlewarePhase::PreReturn,
|
|
)
|
|
.await?
|
|
.entries)
|
|
}
|
|
|
|
async fn describe_chain_for(
|
|
&self,
|
|
entries: &[ChainEntry],
|
|
operation: SupervisorMiddlewareOperation,
|
|
phase: SupervisorMiddlewarePhase,
|
|
) -> Result<DescribedChain> {
|
|
ensure_chain_capacity(entries.len())?;
|
|
let manifests = self.manifests().await?;
|
|
let mut entries = entries.to_vec();
|
|
sort_chain_entries(&mut entries);
|
|
let mut described_entries = Vec::with_capacity(entries.len());
|
|
let mut unbound = Vec::new();
|
|
for entry in entries {
|
|
let Some((state, manifest)) = manifests.iter().find(|(state, manifest)| {
|
|
Self::attachment_name(state, manifest) == entry.implementation
|
|
}) else {
|
|
described_entries.push(DescribedChainEntry {
|
|
entry,
|
|
service: None,
|
|
binding: None,
|
|
max_payload_bytes: 0,
|
|
timeout: DEFAULT_MIDDLEWARE_TIMEOUT,
|
|
});
|
|
continue;
|
|
};
|
|
let Some(binding) = Self::binding(manifest, operation, phase).copied() else {
|
|
// The config remains globally ordered, but it does not
|
|
// participate in this exact operation/phase chain.
|
|
unbound.push(entry);
|
|
continue;
|
|
};
|
|
let timeout = state.timeout_for_binding(&binding)?;
|
|
let advertised = validate_payload_limit("middleware manifest", &binding)?;
|
|
let max_payload_bytes = state.operator_max_payload_bytes.unwrap_or(advertised);
|
|
described_entries.push(DescribedChainEntry {
|
|
entry,
|
|
service: Some(Arc::clone(state)),
|
|
binding: Some(binding),
|
|
max_payload_bytes,
|
|
timeout,
|
|
});
|
|
}
|
|
ensure_chain_capacity(described_entries.len())?;
|
|
Ok(DescribedChain {
|
|
entries: described_entries,
|
|
unbound,
|
|
})
|
|
}
|
|
|
|
pub async fn validate_config(
|
|
&self,
|
|
middleware_name: &str,
|
|
config: prost_types::Struct,
|
|
) -> Result<()> {
|
|
if config.encoded_len() > MAX_MIDDLEWARE_CONFIG_BYTES {
|
|
return Err(miette!(
|
|
"middleware config exceeds the platform maximum of {MAX_MIDDLEWARE_CONFIG_BYTES} encoded bytes"
|
|
));
|
|
}
|
|
let manifests = self.manifests().await?;
|
|
let Some((state, _manifest)) = manifests
|
|
.iter()
|
|
.find(|(state, manifest)| Self::attachment_name(state, manifest) == middleware_name)
|
|
else {
|
|
return Err(miette!("middleware '{middleware_name}' is not registered"));
|
|
};
|
|
let response = call_with_timeout(
|
|
state.operator_timeout,
|
|
"ValidateConfig",
|
|
state.service.validate_config(middleware_name, &config),
|
|
)
|
|
.await
|
|
.map(tonic::Response::into_inner)
|
|
.map_err(|error| {
|
|
miette!(
|
|
"middleware ValidateConfig failed: {}",
|
|
safe_reason(&error.to_string())
|
|
)
|
|
})?;
|
|
if response.valid {
|
|
Ok(())
|
|
} else {
|
|
Err(miette!("{}", safe_reason(&response.reason)))
|
|
}
|
|
}
|
|
|
|
pub async fn evaluate(
|
|
&self,
|
|
entries: &[ChainEntry],
|
|
input: HttpRequestInput,
|
|
) -> Result<ChainOutcome> {
|
|
let entries = self.describe_chain(entries).await?;
|
|
self.evaluate_described(&entries, input).await
|
|
}
|
|
|
|
pub async fn evaluate_described(
|
|
&self,
|
|
entries: &[DescribedChainEntry],
|
|
input: HttpRequestInput,
|
|
) -> Result<ChainOutcome> {
|
|
self.evaluate_described_with_policy(
|
|
entries,
|
|
input,
|
|
TransformedBodyPolicy::NotPolicyRelevant,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// Evaluate a described chain, re-checking the request body against sandbox
|
|
/// policy after every stage that replaces it. Policy runs on the original
|
|
/// body before the chain, so without this a stage could hand the next stage
|
|
/// (or the upstream) a payload the policy rejects. When the evaluator returns
|
|
/// a deny reason the chain stops with that reason, so no later stage ever
|
|
/// sees a non-compliant body. Body-independent protocols must select
|
|
/// [`TransformedBodyPolicy::NotPolicyRelevant`] explicitly.
|
|
pub async fn evaluate_described_with_policy(
|
|
&self,
|
|
entries: &[DescribedChainEntry],
|
|
input: HttpRequestInput,
|
|
transformed_body_policy: TransformedBodyPolicy<'_>,
|
|
) -> Result<ChainOutcome> {
|
|
let admission = if entries.is_empty() {
|
|
None
|
|
} else {
|
|
Some(self.reserve_middleware_work_admission().await?)
|
|
};
|
|
self.evaluate_described_with_policy_admitted(
|
|
entries,
|
|
input,
|
|
transformed_body_policy,
|
|
admission,
|
|
)
|
|
.await
|
|
}
|
|
|
|
/// Evaluate a chain using capacity reserved before its request body was
|
|
/// buffered. The guard is retained until the ordered chain completes.
|
|
pub async fn evaluate_described_with_policy_admitted(
|
|
&self,
|
|
entries: &[DescribedChainEntry],
|
|
input: HttpRequestInput,
|
|
transformed_body_policy: TransformedBodyPolicy<'_>,
|
|
admission: Option<MiddlewareWorkAdmission>,
|
|
) -> Result<ChainOutcome> {
|
|
ensure_chain_capacity(entries.len())?;
|
|
let HttpRequestInput {
|
|
request_id,
|
|
sandbox_id,
|
|
sandbox_name,
|
|
workspace,
|
|
scheme,
|
|
host,
|
|
port,
|
|
method,
|
|
path,
|
|
query,
|
|
headers,
|
|
connection_nominated_headers,
|
|
body,
|
|
} = input;
|
|
// The request envelope is moved into one stable chain state. Built-ins
|
|
// borrow these values for every stage; only the gRPC adapter clones them
|
|
// when an operator service requires an owned protobuf message.
|
|
let context = RequestContext {
|
|
request_id,
|
|
sandbox_id,
|
|
sandbox: sandbox_name,
|
|
workspace,
|
|
originating_process: None,
|
|
};
|
|
let target = HttpRequestTarget {
|
|
scheme,
|
|
host,
|
|
port: u32::from(port),
|
|
method,
|
|
path,
|
|
query,
|
|
};
|
|
let mut headers: Vec<HttpHeader> = headers
|
|
.into_iter()
|
|
.map(|(name, value)| HttpHeader { name, value })
|
|
.collect();
|
|
let mut body = body;
|
|
let mut header_mutations = Vec::new();
|
|
let mut findings = Vec::new();
|
|
let mut metadata = BTreeMap::new();
|
|
let mut applied = Vec::new();
|
|
let _admission = admission;
|
|
let chain_deadline = tokio::time::Instant::now() + MAX_MIDDLEWARE_CHAIN_TIMEOUT;
|
|
|
|
for entry in entries {
|
|
let Some(_binding) = entry.binding.as_ref() else {
|
|
match apply_on_error(entry, "binding_not_described", &mut applied) {
|
|
OnErrorAction::FailOpen => continue,
|
|
OnErrorAction::FailClosed(reason) => {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
};
|
|
if body.len() > entry.max_payload_bytes {
|
|
match apply_on_error(entry, "request_body_over_capacity", &mut applied) {
|
|
OnErrorAction::FailOpen => continue,
|
|
OnErrorAction::FailClosed(reason) => {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
let request = HttpRequestView::new(
|
|
PRE_CREDENTIALS_PHASE,
|
|
&context,
|
|
&entry.entry.config,
|
|
&target,
|
|
&headers,
|
|
&body,
|
|
&entry.entry.implementation,
|
|
);
|
|
if let Err(reason) = validate_request_view(request) {
|
|
match apply_on_error(entry, reason, &mut applied) {
|
|
OnErrorAction::FailOpen => continue,
|
|
OnErrorAction::FailClosed(reason) => {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
let Some(service) = entry.service.as_ref() else {
|
|
unreachable!("described binding always has a service")
|
|
};
|
|
let remaining = chain_deadline.saturating_duration_since(tokio::time::Instant::now());
|
|
if remaining.is_zero() {
|
|
match apply_on_error(entry, "middleware_chain_timeout", &mut applied) {
|
|
OnErrorAction::FailOpen => continue,
|
|
OnErrorAction::FailClosed(reason) => {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
let mut result = match call_with_timeout(
|
|
entry.timeout.min(remaining),
|
|
"EvaluateHttpRequest",
|
|
service.service.evaluate_http_request(request),
|
|
)
|
|
.await
|
|
{
|
|
Ok(result) => result.into_inner(),
|
|
Err(err) => {
|
|
let reason = if err.code() == tonic::Code::DeadlineExceeded {
|
|
"middleware_timeout".to_string()
|
|
} else {
|
|
service.diagnostic_policy.error_reason(&err)
|
|
};
|
|
match apply_on_error(entry, &reason, &mut applied) {
|
|
OnErrorAction::FailOpen => continue,
|
|
OnErrorAction::FailClosed(reason) => {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
if let Err(reason) = validate_response_envelope(&result) {
|
|
match apply_on_error(entry, reason, &mut applied) {
|
|
OnErrorAction::FailOpen => continue,
|
|
OnErrorAction::FailClosed(reason) => {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
service
|
|
.diagnostic_policy
|
|
.process_result(&entry.entry.implementation, &mut result);
|
|
|
|
let decision = match Decision::try_from(result.decision) {
|
|
Ok(decision @ (Decision::Allow | Decision::Deny)) => decision,
|
|
Ok(Decision::Unspecified) | Err(_) => {
|
|
match apply_on_error(entry, "invalid_response_decision", &mut applied) {
|
|
OnErrorAction::FailOpen => continue,
|
|
OnErrorAction::FailClosed(reason) => {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
if decision == Decision::Deny {
|
|
let reason_code =
|
|
(!result.reason_code.is_empty()).then(|| result.reason_code.clone());
|
|
let denial = MiddlewareDenial {
|
|
config_name: entry.entry.name.clone(),
|
|
reason_code,
|
|
};
|
|
for finding in result.findings {
|
|
findings.push(NamespacedFinding {
|
|
middleware: entry.entry.name.clone(),
|
|
finding,
|
|
});
|
|
}
|
|
if !result.metadata.is_empty() {
|
|
metadata.insert(
|
|
entry.entry.name.clone(),
|
|
result.metadata.into_iter().collect(),
|
|
);
|
|
}
|
|
applied.push(MiddlewareInvocation {
|
|
name: entry.entry.name.clone(),
|
|
implementation: entry.entry.implementation.clone(),
|
|
decision,
|
|
transformed: false,
|
|
failed: false,
|
|
});
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason: middleware_denial_reason(
|
|
&denial.config_name,
|
|
denial.reason_code.as_deref(),
|
|
),
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: Some(denial),
|
|
});
|
|
}
|
|
|
|
if result.has_body && result.body.len() > entry.max_payload_bytes {
|
|
match apply_on_error(entry, "response_body_over_capacity", &mut applied) {
|
|
OnErrorAction::FailOpen => continue,
|
|
OnErrorAction::FailClosed(reason) => {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
// Validate and apply the entire stage atomically. Under fail-open,
|
|
// one malformed mutation must not leave earlier mutations from the
|
|
// same response visible to later middleware.
|
|
let updated_headers = if result.header_mutations.is_empty() {
|
|
None
|
|
} else {
|
|
match headers::apply(
|
|
headers::HeaderAuthority::Request,
|
|
&headers,
|
|
&connection_nominated_headers,
|
|
&result.header_mutations,
|
|
) {
|
|
Ok(updated) => Some(updated),
|
|
Err(error) => {
|
|
let reason = service
|
|
.diagnostic_policy
|
|
.header_mutation_error_reason(&error);
|
|
match apply_on_error(entry, &reason, &mut applied) {
|
|
OnErrorAction::FailOpen => continue,
|
|
OnErrorAction::FailClosed(reason) => {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
}
|
|
};
|
|
let headers_transformed = updated_headers
|
|
.as_ref()
|
|
.is_some_and(|updated| updated != &headers);
|
|
if let Some(updated) = updated_headers {
|
|
headers = updated;
|
|
}
|
|
header_mutations.extend(std::mem::take(&mut result.header_mutations));
|
|
|
|
let body_transformed = result.has_body;
|
|
if body_transformed {
|
|
body = std::mem::take(&mut result.body);
|
|
}
|
|
for finding in result.findings {
|
|
findings.push(NamespacedFinding {
|
|
middleware: entry.entry.name.clone(),
|
|
finding,
|
|
});
|
|
}
|
|
if !result.metadata.is_empty() {
|
|
metadata.insert(
|
|
entry.entry.name.clone(),
|
|
result.metadata.into_iter().collect(),
|
|
);
|
|
}
|
|
applied.push(MiddlewareInvocation {
|
|
name: entry.entry.name.clone(),
|
|
implementation: entry.entry.implementation.clone(),
|
|
decision,
|
|
transformed: body_transformed || headers_transformed,
|
|
failed: false,
|
|
});
|
|
|
|
// The stage ran successfully but its output must still satisfy the
|
|
// sandbox policy the original body was admitted under. Re-check now,
|
|
// before the next stage or the upstream sees the replaced body. A
|
|
// policy deny here is a hard deny, independent of `on_error`.
|
|
if body_transformed
|
|
&& let TransformedBodyPolicy::Reevaluate(validate) = transformed_body_policy
|
|
{
|
|
let denied = match validate(&body) {
|
|
Ok(reason) => reason,
|
|
Err(error) => Some(format!(
|
|
"transformed_body_policy_evaluation_failed: {}",
|
|
safe_reason(&error.to_string())
|
|
)),
|
|
};
|
|
if let Some(reason) = denied {
|
|
return Ok(ChainOutcome {
|
|
allowed: false,
|
|
reason,
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
Ok(ChainOutcome {
|
|
allowed: true,
|
|
reason: String::new(),
|
|
body,
|
|
header_mutations,
|
|
findings,
|
|
metadata,
|
|
applied,
|
|
denial: None,
|
|
})
|
|
}
|
|
}
|
|
|
|
/// Sort middleware by policy-defined priority. Valid policies have unique order
|
|
/// values; the name comparison only keeps direct internal callers deterministic.
|
|
pub fn sort_chain_entries(entries: &mut [ChainEntry]) {
|
|
entries.sort_by(|left, right| {
|
|
left.order
|
|
.cmp(&right.order)
|
|
.then_with(|| left.name.cmp(&right.name))
|
|
});
|
|
}
|
|
|
|
fn ensure_config_capacity(count: usize) -> Result<()> {
|
|
if count > MAX_MIDDLEWARE_CONFIGS {
|
|
return Err(miette!(
|
|
"middleware config count {count} exceeds platform maximum {MAX_MIDDLEWARE_CONFIGS}"
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn ensure_chain_capacity(count: usize) -> Result<()> {
|
|
if count > MAX_MIDDLEWARE_CHAIN_STAGES {
|
|
return Err(miette!(
|
|
"selected middleware stage count {count} exceeds platform maximum {MAX_MIDDLEWARE_CHAIN_STAGES}"
|
|
));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
pub(crate) fn safe_reason(reason: &str) -> String {
|
|
reason
|
|
.chars()
|
|
.filter(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '_' | '-' | ':' | ' '))
|
|
.take(160)
|
|
.collect()
|
|
}
|
|
|
|
#[cfg(test)]
|
|
mod tests {
|
|
use super::*;
|
|
use futures::{FutureExt, Stream, StreamExt};
|
|
use openshell_core::proto::middleware::v1::supervisor_middleware_server::{
|
|
SupervisorMiddleware, SupervisorMiddlewareServer,
|
|
};
|
|
use openshell_core::proto::{ExistingHeaderAction, header_mutation};
|
|
use openshell_supervisor_middleware_builtins::{BUILTIN_REGEX, services};
|
|
|
|
use tokio_stream::wrappers::TcpListenerStream;
|
|
|
|
fn proto_duration(value: &str) -> prost_types::Duration {
|
|
let duration = match (value.strip_suffix("ms"), value.strip_suffix('s')) {
|
|
(Some(milliseconds), _) => {
|
|
Duration::from_millis(milliseconds.parse().expect("integer milliseconds"))
|
|
}
|
|
(_, Some(seconds)) => Duration::from_secs(seconds.parse().expect("integer seconds")),
|
|
(None, None) => panic!("test duration must use ms or s"),
|
|
};
|
|
openshell_core::time::duration_from_std(duration)
|
|
.expect("test duration is in protobuf range")
|
|
}
|
|
|
|
#[test]
|
|
fn advertised_audience_mismatch_fails_registration() {
|
|
let configured = "urn:openshell:extension:middleware:content-guard";
|
|
|
|
// Matching and unadvertised audiences both pass.
|
|
validate_expected_audience("content-guard", configured, "", true)
|
|
.expect("unadvertised audience is accepted");
|
|
validate_expected_audience("content-guard", configured, configured, true)
|
|
.expect("matching audience is accepted");
|
|
|
|
// Once authenticated Describe succeeds, reject a manifest that
|
|
// contradicts the operator-owned audience configuration.
|
|
let error =
|
|
validate_expected_audience("content-guard", configured, "urn:example:stale", true)
|
|
.expect_err("mismatched audience must fail closed");
|
|
let message = error.to_string();
|
|
assert!(message.contains("urn:example:stale"));
|
|
assert!(message.contains(configured));
|
|
|
|
// The check does not apply where no credential is attached at all,
|
|
// whether because the registration opted out or because the gateway
|
|
// has no signing key configured.
|
|
validate_expected_audience("content-guard", configured, "urn:example:stale", false)
|
|
.expect("an unauthenticated call has no audience to mismatch");
|
|
}
|
|
|
|
fn builtin_runner() -> ChainRunner {
|
|
ChainRunner::new(
|
|
services()
|
|
.into_iter()
|
|
.next()
|
|
.expect("built-in middleware service"),
|
|
)
|
|
}
|
|
|
|
fn entry(name: &str, on_error: OnError) -> ChainEntry {
|
|
ChainEntry {
|
|
name: name.into(),
|
|
implementation: BUILTIN_REGEX.into(),
|
|
order: 0,
|
|
config: prost_types::Struct {
|
|
fields: std::iter::once((
|
|
"mode".into(),
|
|
prost_types::Value {
|
|
kind: Some(prost_types::value::Kind::StringValue("redact".into())),
|
|
},
|
|
))
|
|
.collect(),
|
|
},
|
|
on_error,
|
|
}
|
|
}
|
|
|
|
fn input(body: &str) -> HttpRequestInput {
|
|
HttpRequestInput {
|
|
request_id: "req".into(),
|
|
sandbox_id: "sbx-id".into(),
|
|
sandbox_name: "sbx-name".into(),
|
|
workspace: "wrks-default".into(),
|
|
scheme: "https".into(),
|
|
host: "api.example.com".into(),
|
|
port: 443,
|
|
method: "POST".into(),
|
|
path: "/v1".into(),
|
|
query: String::new(),
|
|
headers: Vec::new(),
|
|
connection_nominated_headers: Vec::new(),
|
|
body: body.as_bytes().to_vec(),
|
|
}
|
|
}
|
|
|
|
fn write_header(name: &str, value: &str, on_existing: ExistingHeaderAction) -> HeaderMutation {
|
|
HeaderMutation {
|
|
operation: Some(header_mutation::Operation::Write(
|
|
openshell_core::proto::WriteHeader {
|
|
name: name.into(),
|
|
value: value.into(),
|
|
on_existing: on_existing as i32,
|
|
},
|
|
)),
|
|
}
|
|
}
|
|
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
struct RequestAddresses {
|
|
phase: SupervisorMiddlewarePhase,
|
|
context: usize,
|
|
request_id: usize,
|
|
config: usize,
|
|
target: usize,
|
|
host: usize,
|
|
headers: usize,
|
|
first_header_name: usize,
|
|
body: usize,
|
|
originating_process_present: bool,
|
|
middleware_name: String,
|
|
}
|
|
|
|
/// Records borrowed addresses so the test can detect an owned envelope
|
|
/// being reconstructed between otherwise no-op in-process stages.
|
|
struct BorrowedRecordingService {
|
|
manifest_name: String,
|
|
received: std::sync::Mutex<Vec<RequestAddresses>>,
|
|
advertise_protocol: bool,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl InProcessMiddleware for BorrowedRecordingService {
|
|
async fn describe(&self) -> MiddlewareManifest {
|
|
MiddlewareManifest {
|
|
name: self.manifest_name.clone(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: self.advertise_protocol.then(|| {
|
|
openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)
|
|
}),
|
|
}
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
_middleware_name: &str,
|
|
_config: &prost_types::Struct,
|
|
) -> Result<()> {
|
|
Ok(())
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: HttpRequestView<'_>,
|
|
) -> Result<openshell_core::proto::HttpRequestResult> {
|
|
let addresses = RequestAddresses {
|
|
phase: request.phase(),
|
|
context: std::ptr::from_ref(request.context()).addr(),
|
|
request_id: request.context().request_id.as_ptr().addr(),
|
|
config: std::ptr::from_ref(request.config()).addr(),
|
|
target: std::ptr::from_ref(request.target()).addr(),
|
|
host: request.target().host.as_ptr().addr(),
|
|
headers: request.headers().as_ptr().addr(),
|
|
first_header_name: request
|
|
.headers()
|
|
.first()
|
|
.map_or(0, |header| header.name.as_ptr().addr()),
|
|
body: request.body().as_ptr().addr(),
|
|
originating_process_present: request.context().originating_process.is_some(),
|
|
middleware_name: request.middleware_name().to_string(),
|
|
};
|
|
self.received
|
|
.lock()
|
|
.expect("borrowed request recorder lock")
|
|
.push(addresses);
|
|
Ok(allow_result())
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn in_process_stages_share_one_borrowed_request_envelope() {
|
|
let service = Arc::new(BorrowedRecordingService {
|
|
manifest_name: "acme/redactor".into(),
|
|
received: std::sync::Mutex::new(Vec::new()),
|
|
advertise_protocol: true,
|
|
});
|
|
let runner = ChainRunner::new(service.clone());
|
|
let entries = [
|
|
ChainEntry {
|
|
name: "first".into(),
|
|
implementation: "acme/redactor".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
ChainEntry {
|
|
name: "second".into(),
|
|
implementation: "acme/redactor".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
];
|
|
let described = runner
|
|
.describe_chain(&entries)
|
|
.await
|
|
.expect("describe chain");
|
|
let expected_configs: Vec<_> = described
|
|
.iter()
|
|
.map(|entry| std::ptr::from_ref(&entry.entry.config).addr())
|
|
.collect();
|
|
let mut request = input("payload");
|
|
request.headers = vec![("x-test".into(), "value".into())];
|
|
let expected_body = request.body.as_ptr().addr();
|
|
let expected_request_id = request.request_id.as_ptr().addr();
|
|
let expected_host = request.host.as_ptr().addr();
|
|
let expected_header_name = request.headers[0].0.as_ptr().addr();
|
|
|
|
let outcome = runner
|
|
.evaluate_described(&described, request)
|
|
.await
|
|
.expect("evaluate borrowed chain");
|
|
let received = service.received.lock().expect("borrowed requests");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.body.as_ptr().addr(), expected_body);
|
|
assert_eq!(received.len(), 2);
|
|
assert_eq!(received[0].phase, SupervisorMiddlewarePhase::PreCredentials);
|
|
assert!(!received[0].originating_process_present);
|
|
assert_eq!(received[0].request_id, expected_request_id);
|
|
assert_eq!(received[0].host, expected_host);
|
|
assert_eq!(received[0].first_header_name, expected_header_name);
|
|
assert_eq!(received[0].body, expected_body);
|
|
assert_eq!(received[0].config, expected_configs[0]);
|
|
assert_eq!(received[1].config, expected_configs[1]);
|
|
assert_eq!(received[0].context, received[1].context);
|
|
assert_eq!(received[0].target, received[1].target);
|
|
assert_eq!(received[0].headers, received[1].headers);
|
|
assert_eq!(received[0].body, received[1].body);
|
|
assert!(
|
|
received
|
|
.iter()
|
|
.all(|request| request.middleware_name == "acme/redactor")
|
|
);
|
|
}
|
|
|
|
const TEST_REPLACEMENT_BODY: &[u8] = b"stage-one-replacement";
|
|
|
|
/// Records both sides of a successful body replacement so the test can
|
|
/// distinguish ownership transfer from a content-preserving body copy.
|
|
#[derive(Debug, Default)]
|
|
struct ReplacementTransferRecord {
|
|
invocations: usize,
|
|
returned_body: Option<usize>,
|
|
second_body: Option<usize>,
|
|
second_body_bytes: Vec<u8>,
|
|
}
|
|
|
|
/// Replaces the first request body and observes the body borrowed by the
|
|
/// second stage without replacing it again.
|
|
struct ReplacementTransferService {
|
|
record: std::sync::Mutex<ReplacementTransferRecord>,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl InProcessMiddleware for ReplacementTransferService {
|
|
async fn describe(&self) -> MiddlewareManifest {
|
|
MiddlewareManifest {
|
|
name: "test/replacement-transfer".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
}
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
_middleware_name: &str,
|
|
_config: &prost_types::Struct,
|
|
) -> Result<()> {
|
|
Ok(())
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: HttpRequestView<'_>,
|
|
) -> Result<openshell_core::proto::HttpRequestResult> {
|
|
let mut record = self.record.lock().expect("replacement transfer record");
|
|
let invocation = record.invocations;
|
|
record.invocations += 1;
|
|
|
|
if invocation == 0 {
|
|
let replacement = TEST_REPLACEMENT_BODY.to_vec();
|
|
record.returned_body = Some(replacement.as_ptr().addr());
|
|
let mut result = allow_result();
|
|
result.body = replacement;
|
|
result.has_body = true;
|
|
Ok(result)
|
|
} else {
|
|
record.second_body = Some(request.body().as_ptr().addr());
|
|
record.second_body_bytes = request.body().to_vec();
|
|
Ok(allow_result())
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn replacement_body_allocation_moves_through_next_stage_and_outcome() {
|
|
let service = Arc::new(ReplacementTransferService {
|
|
record: std::sync::Mutex::new(ReplacementTransferRecord::default()),
|
|
});
|
|
let runner = ChainRunner::new(service.clone());
|
|
let entries = [
|
|
ChainEntry {
|
|
name: "replace".into(),
|
|
implementation: "test/replacement-transfer".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
ChainEntry {
|
|
name: "observe".into(),
|
|
implementation: "test/replacement-transfer".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
];
|
|
|
|
let outcome = runner
|
|
.evaluate(&entries, input("original-body"))
|
|
.await
|
|
.expect("evaluate replacement transfer chain");
|
|
let record = service.record.lock().expect("replacement transfer record");
|
|
let returned_body = record
|
|
.returned_body
|
|
.expect("first-stage replacement pointer");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(record.invocations, 2);
|
|
assert_eq!(record.second_body_bytes, TEST_REPLACEMENT_BODY);
|
|
assert_eq!(record.second_body, Some(returned_body));
|
|
assert_eq!(outcome.body, TEST_REPLACEMENT_BODY);
|
|
assert_eq!(outcome.body.as_ptr().addr(), returned_body);
|
|
}
|
|
|
|
/// An in-process service that yields forever so the runtime must enforce
|
|
/// the binding timeout around borrowed validation and evaluation futures.
|
|
struct PendingInProcessService;
|
|
|
|
#[tonic::async_trait]
|
|
impl InProcessMiddleware for PendingInProcessService {
|
|
async fn describe(&self) -> MiddlewareManifest {
|
|
MiddlewareManifest {
|
|
name: "test/pending".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: Some(proto_duration("10ms")),
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
}
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
_middleware_name: &str,
|
|
_config: &prost_types::Struct,
|
|
) -> Result<()> {
|
|
std::future::pending().await
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
_request: HttpRequestView<'_>,
|
|
) -> Result<openshell_core::proto::HttpRequestResult> {
|
|
std::future::pending().await
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn in_process_evaluation_remains_interruptible_by_stage_timeout() {
|
|
let runner = ChainRunner::new(Arc::new(PendingInProcessService));
|
|
let entry = |on_error| ChainEntry {
|
|
name: "pending".into(),
|
|
implementation: "test/pending".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error,
|
|
};
|
|
let closed = runner
|
|
.evaluate(&[entry(OnError::FailClosed)], input("payload"))
|
|
.await
|
|
.expect("timed-out in-process evaluation");
|
|
let open = runner
|
|
.evaluate(&[entry(OnError::FailOpen)], input("payload"))
|
|
.await
|
|
.expect("fail-open timed-out in-process evaluation");
|
|
|
|
assert!(!closed.allowed);
|
|
assert_eq!(closed.reason, "middleware_failed: middleware_timeout");
|
|
assert!(closed.applied[0].failed);
|
|
assert!(open.allowed);
|
|
assert!(open.applied[0].failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn in_process_validation_remains_interruptible_by_binding_timeout() {
|
|
let runner = ChainRunner::new(Arc::new(PendingInProcessService));
|
|
let error = runner
|
|
.validate_config("test/pending", prost_types::Struct::default())
|
|
.await
|
|
.expect_err("timed-out in-process validation");
|
|
|
|
assert!(error.to_string().contains("ValidateConfig failed"));
|
|
assert!(error.to_string().contains("timed out"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn applies_fixed_regex_replacements() {
|
|
let outcome = builtin_runner()
|
|
.evaluate(
|
|
&[entry("redact", OnError::FailClosed)],
|
|
input(r#"{"api_key":"sk-1234567890abcdef"}"#),
|
|
)
|
|
.await
|
|
.expect("evaluate");
|
|
assert!(outcome.allowed);
|
|
assert_eq!(
|
|
String::from_utf8(outcome.body).expect("utf8"),
|
|
r#"{"api_key":"[REDACTED]"}"#
|
|
);
|
|
assert_eq!(outcome.findings[0].finding.count, 1);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn transformed_body_feeds_next_stage() {
|
|
let entries = [
|
|
entry("first", OnError::FailClosed),
|
|
entry("second", OnError::FailClosed),
|
|
];
|
|
let outcome = builtin_runner()
|
|
.evaluate(&entries, input(r#"token="sk-ABCDEFGHIJKLMNOP""#))
|
|
.await
|
|
.expect("evaluate");
|
|
assert!(outcome.allowed);
|
|
assert_eq!(
|
|
String::from_utf8(outcome.body).expect("utf8"),
|
|
r#"token="[REDACTED]""#
|
|
);
|
|
assert_eq!(outcome.applied.len(), 2);
|
|
assert_eq!(
|
|
[
|
|
outcome.applied[0].transformed,
|
|
outcome.applied[1].transformed,
|
|
],
|
|
[true, false]
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn describe_chain_sorts_by_order_then_name() {
|
|
let mut later = entry("later", OnError::FailClosed);
|
|
later.order = 20;
|
|
let mut beta = entry("beta", OnError::FailClosed);
|
|
beta.order = 10;
|
|
let mut alpha = entry("alpha", OnError::FailClosed);
|
|
alpha.order = 10;
|
|
|
|
let described = builtin_runner()
|
|
.describe_chain(&[later, beta, alpha])
|
|
.await
|
|
.expect("describe ordered chain");
|
|
let names: Vec<_> = described
|
|
.iter()
|
|
.map(|entry| entry.entry.name.as_str())
|
|
.collect();
|
|
assert_eq!(names, vec!["alpha", "beta", "later"]);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn describe_chain_accepts_maximum_selected_stages() {
|
|
let entries: Vec<_> = (0..MAX_MIDDLEWARE_CHAIN_STAGES)
|
|
.map(|index| entry(&format!("stage-{index}"), OnError::FailClosed))
|
|
.collect();
|
|
|
|
let described = builtin_runner()
|
|
.describe_chain(&entries)
|
|
.await
|
|
.expect("maximum selected stage count");
|
|
assert_eq!(described.len(), MAX_MIDDLEWARE_CHAIN_STAGES);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn describe_chain_rejects_selected_stages_over_capacity() {
|
|
let entries: Vec<_> = (0..=MAX_MIDDLEWARE_CHAIN_STAGES)
|
|
.map(|index| entry(&format!("stage-{index}"), OnError::FailClosed))
|
|
.collect();
|
|
|
|
let error = builtin_runner()
|
|
.describe_chain(&entries)
|
|
.await
|
|
.err()
|
|
.expect("selected stage count over capacity");
|
|
assert!(
|
|
error
|
|
.to_string()
|
|
.contains("selected middleware stage count 11 exceeds platform maximum 10")
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn fail_open_allows_unavailable_middleware() {
|
|
let unavailable = ChainEntry {
|
|
name: "missing".into(),
|
|
implementation: "third-party/missing".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailOpen,
|
|
};
|
|
let outcome = builtin_runner()
|
|
.evaluate(&[unavailable], input("hello"))
|
|
.await
|
|
.expect("evaluate");
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.body, b"hello");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn fail_closed_denies_unavailable_middleware() {
|
|
let unavailable = ChainEntry {
|
|
name: "missing".into(),
|
|
implementation: "third-party/missing".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
let outcome = builtin_runner()
|
|
.evaluate(&[unavailable], input("hello"))
|
|
.await
|
|
.expect("evaluate");
|
|
assert!(!outcome.allowed);
|
|
assert!(outcome.reason.starts_with("middleware_failed:"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn injected_service_names_drive_registration_checks() {
|
|
let registry = MiddlewareRegistry::connect_services(services(), Vec::new())
|
|
.await
|
|
.expect("connect built-in service");
|
|
let policy = SandboxPolicy {
|
|
network_middlewares: HashMap::from([(
|
|
"redactor".into(),
|
|
NetworkMiddlewareConfig {
|
|
middleware: BUILTIN_REGEX.into(),
|
|
..Default::default()
|
|
},
|
|
)]),
|
|
..Default::default()
|
|
};
|
|
registry
|
|
.ensure_policy_middlewares_registered(&policy)
|
|
.expect("described middleware is registered");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn injected_services_cannot_duplicate_middleware_names() {
|
|
let first: Arc<dyn InProcessMiddleware> = Arc::new(BorrowedRecordingService {
|
|
manifest_name: "openshell/test".into(),
|
|
received: std::sync::Mutex::new(Vec::new()),
|
|
advertise_protocol: true,
|
|
});
|
|
let second: Arc<dyn InProcessMiddleware> = Arc::new(BorrowedRecordingService {
|
|
manifest_name: "openshell/test".into(),
|
|
received: std::sync::Mutex::new(Vec::new()),
|
|
advertise_protocol: true,
|
|
});
|
|
|
|
let error = MiddlewareRegistry::connect_services(vec![first, second], Vec::new())
|
|
.await
|
|
.expect_err("duplicate injected middleware name must fail registry construction");
|
|
assert!(
|
|
error
|
|
.to_string()
|
|
.contains("duplicate supervisor middleware name")
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn in_process_service_without_protocol_metadata_is_rejected() {
|
|
let service: Arc<dyn InProcessMiddleware> = Arc::new(BorrowedRecordingService {
|
|
manifest_name: "openshell/legacy".into(),
|
|
received: std::sync::Mutex::new(Vec::new()),
|
|
advertise_protocol: false,
|
|
});
|
|
|
|
let error = MiddlewareRegistry::connect_services(vec![service], Vec::new())
|
|
.await
|
|
.expect_err("legacy middleware must fail negotiation");
|
|
|
|
assert!(
|
|
error
|
|
.to_string()
|
|
.contains("did not provide protocol metadata")
|
|
);
|
|
}
|
|
|
|
/// A mock middleware that returns a fixed, caller-supplied result for every
|
|
/// evaluation. Used to exercise chain behavior the built-in cannot produce
|
|
/// (explicit deny, metadata, findings, unsafe header mutations).
|
|
struct ScriptedService {
|
|
manifest_name: String,
|
|
max_body_bytes: u64,
|
|
result: openshell_core::proto::HttpRequestResult,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl SupervisorMiddleware for ScriptedService {
|
|
type EvaluateWebSocketSessionStream = WebSocketResponseStream;
|
|
|
|
async fn evaluate_web_socket_session(
|
|
&self,
|
|
_request: Request<tonic::Streaming<openshell_core::proto::WebSocketSessionEvent>>,
|
|
) -> std::result::Result<tonic::Response<Self::EvaluateWebSocketSessionStream>, tonic::Status>
|
|
{
|
|
Err(tonic::Status::unimplemented("HTTP-only test middleware"))
|
|
}
|
|
|
|
async fn describe(
|
|
&self,
|
|
_request: Request<MiddlewareDescribeRequest>,
|
|
) -> std::result::Result<tonic::Response<MiddlewareManifest>, tonic::Status> {
|
|
Ok(tonic::Response::new(MiddlewareManifest {
|
|
name: self.manifest_name.clone(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: self.max_body_bytes,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
}))
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
_request: Request<ValidateConfigRequest>,
|
|
) -> std::result::Result<tonic::Response<ValidateConfigResponse>, tonic::Status> {
|
|
Ok(tonic::Response::new(ValidateConfigResponse {
|
|
valid: true,
|
|
reason: String::new(),
|
|
}))
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
_request: Request<HttpRequestEvaluation>,
|
|
) -> std::result::Result<
|
|
tonic::Response<openshell_core::proto::HttpRequestResult>,
|
|
tonic::Status,
|
|
> {
|
|
Ok(tonic::Response::new(self.result.clone()))
|
|
}
|
|
}
|
|
|
|
struct SlowService {
|
|
delay: Duration,
|
|
binding_timeout: Option<prost_types::Duration>,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl SupervisorMiddleware for SlowService {
|
|
type EvaluateWebSocketSessionStream = WebSocketResponseStream;
|
|
|
|
async fn evaluate_web_socket_session(
|
|
&self,
|
|
_request: Request<tonic::Streaming<openshell_core::proto::WebSocketSessionEvent>>,
|
|
) -> std::result::Result<tonic::Response<Self::EvaluateWebSocketSessionStream>, tonic::Status>
|
|
{
|
|
Err(tonic::Status::unimplemented("HTTP-only test middleware"))
|
|
}
|
|
|
|
async fn describe(
|
|
&self,
|
|
_request: Request<MiddlewareDescribeRequest>,
|
|
) -> std::result::Result<tonic::Response<MiddlewareManifest>, tonic::Status> {
|
|
Ok(tonic::Response::new(MiddlewareManifest {
|
|
name: "test/slow".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: self.binding_timeout,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
}))
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
_request: Request<ValidateConfigRequest>,
|
|
) -> std::result::Result<tonic::Response<ValidateConfigResponse>, tonic::Status> {
|
|
tokio::time::sleep(self.delay).await;
|
|
Ok(tonic::Response::new(ValidateConfigResponse {
|
|
valid: true,
|
|
reason: String::new(),
|
|
}))
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
_request: Request<HttpRequestEvaluation>,
|
|
) -> std::result::Result<
|
|
tonic::Response<openshell_core::proto::HttpRequestResult>,
|
|
tonic::Status,
|
|
> {
|
|
tokio::time::sleep(self.delay).await;
|
|
Ok(tonic::Response::new(allow_result()))
|
|
}
|
|
}
|
|
|
|
/// A middleware attached twice for exercising per-stage validation. The
|
|
/// first policy config requests a body transformation; the second records
|
|
/// that it ran and allows.
|
|
struct TwoStageService {
|
|
second_ran: Arc<std::sync::atomic::AtomicBool>,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl SupervisorMiddleware for TwoStageService {
|
|
type EvaluateWebSocketSessionStream = WebSocketResponseStream;
|
|
|
|
async fn evaluate_web_socket_session(
|
|
&self,
|
|
_request: Request<tonic::Streaming<openshell_core::proto::WebSocketSessionEvent>>,
|
|
) -> std::result::Result<tonic::Response<Self::EvaluateWebSocketSessionStream>, tonic::Status>
|
|
{
|
|
Err(tonic::Status::unimplemented("HTTP-only test middleware"))
|
|
}
|
|
|
|
async fn describe(
|
|
&self,
|
|
_request: Request<MiddlewareDescribeRequest>,
|
|
) -> std::result::Result<tonic::Response<MiddlewareManifest>, tonic::Status> {
|
|
Ok(tonic::Response::new(MiddlewareManifest {
|
|
name: "test/two-stage".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 256 * 1024,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
}))
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
_request: Request<ValidateConfigRequest>,
|
|
) -> std::result::Result<tonic::Response<ValidateConfigResponse>, tonic::Status> {
|
|
Ok(tonic::Response::new(ValidateConfigResponse {
|
|
valid: true,
|
|
reason: String::new(),
|
|
}))
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: Request<HttpRequestEvaluation>,
|
|
) -> std::result::Result<
|
|
tonic::Response<openshell_core::proto::HttpRequestResult>,
|
|
tonic::Status,
|
|
> {
|
|
let evaluation = request.into_inner();
|
|
let mut result = allow_result();
|
|
if evaluation.config.as_ref().is_some_and(|config| {
|
|
config.fields.get("transform").is_some_and(|value| {
|
|
matches!(
|
|
value.kind.as_ref(),
|
|
Some(prost_types::value::Kind::BoolValue(true))
|
|
)
|
|
})
|
|
}) {
|
|
result.body = b"TRANSFORMED".to_vec();
|
|
result.has_body = true;
|
|
} else {
|
|
self.second_ran
|
|
.store(true, std::sync::atomic::Ordering::SeqCst);
|
|
}
|
|
Ok(tonic::Response::new(result))
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn per_stage_validation_denies_before_the_next_stage_runs() {
|
|
// The validator rejects the first stage's transformed body. The chain
|
|
// must stop there: the second stage never runs, so it never sees a
|
|
// payload the policy would reject.
|
|
let second_ran = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
|
let service: Arc<MiddlewareService> = Arc::new(TwoStageService {
|
|
second_ran: Arc::clone(&second_ran),
|
|
});
|
|
let runner = ChainRunner::new_protobuf_for_tests(service);
|
|
let transform = ChainEntry {
|
|
name: "transform".into(),
|
|
implementation: "test/two-stage".into(),
|
|
order: 0,
|
|
config: prost_types::Struct {
|
|
fields: std::iter::once((
|
|
"transform".into(),
|
|
prost_types::Value {
|
|
kind: Some(prost_types::value::Kind::BoolValue(true)),
|
|
},
|
|
))
|
|
.collect(),
|
|
},
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
let second = ChainEntry {
|
|
name: "second".into(),
|
|
implementation: "test/two-stage".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
let described = runner
|
|
.describe_chain(&[transform, second])
|
|
.await
|
|
.expect("describe two-stage chain");
|
|
|
|
let validator: Box<TransformedBodyValidator<'_>> = Box::new(|body: &[u8]| {
|
|
if body == b"TRANSFORMED" {
|
|
Ok(Some("transformed body denied by policy".to_string()))
|
|
} else {
|
|
Ok(None)
|
|
}
|
|
});
|
|
let outcome = runner
|
|
.evaluate_described_with_policy(
|
|
&described,
|
|
input("original"),
|
|
TransformedBodyPolicy::Reevaluate(&*validator),
|
|
)
|
|
.await
|
|
.expect("evaluate two-stage chain");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(outcome.reason, "transformed body denied by policy");
|
|
assert_eq!(outcome.applied.len(), 1, "only the first stage should run");
|
|
assert_eq!(outcome.applied[0].name, "transform");
|
|
assert!(
|
|
!second_ran.load(std::sync::atomic::Ordering::SeqCst),
|
|
"second stage must not run after a policy deny"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn per_stage_validator_allows_compliant_transformations() {
|
|
// A validator that accepts every body lets both stages run; the second
|
|
// stage sees the first stage's output.
|
|
let second_ran = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
|
let service: Arc<MiddlewareService> = Arc::new(TwoStageService {
|
|
second_ran: Arc::clone(&second_ran),
|
|
});
|
|
let runner = ChainRunner::new_protobuf_for_tests(service);
|
|
let transform = ChainEntry {
|
|
name: "transform".into(),
|
|
implementation: "test/two-stage".into(),
|
|
order: 0,
|
|
config: prost_types::Struct {
|
|
fields: std::iter::once((
|
|
"transform".into(),
|
|
prost_types::Value {
|
|
kind: Some(prost_types::value::Kind::BoolValue(true)),
|
|
},
|
|
))
|
|
.collect(),
|
|
},
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
let second = ChainEntry {
|
|
name: "second".into(),
|
|
implementation: "test/two-stage".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
let described = runner
|
|
.describe_chain(&[transform, second])
|
|
.await
|
|
.expect("describe two-stage chain");
|
|
|
|
let validator: Box<TransformedBodyValidator<'_>> = Box::new(|_body: &[u8]| Ok(None));
|
|
let outcome = runner
|
|
.evaluate_described_with_policy(
|
|
&described,
|
|
input("original"),
|
|
TransformedBodyPolicy::Reevaluate(&*validator),
|
|
)
|
|
.await
|
|
.expect("evaluate two-stage chain");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.applied.len(), 2);
|
|
assert!(
|
|
second_ran.load(std::sync::atomic::Ordering::SeqCst),
|
|
"second stage should run when the transformation is compliant"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn per_stage_validator_error_becomes_structured_denial() {
|
|
let second_ran = Arc::new(std::sync::atomic::AtomicBool::new(false));
|
|
let service: Arc<MiddlewareService> = Arc::new(TwoStageService {
|
|
second_ran: Arc::clone(&second_ran),
|
|
});
|
|
let runner = ChainRunner::new_protobuf_for_tests(service);
|
|
let entries = [
|
|
ChainEntry {
|
|
name: "transform".into(),
|
|
implementation: "test/two-stage".into(),
|
|
order: 0,
|
|
config: prost_types::Struct {
|
|
fields: std::iter::once((
|
|
"transform".into(),
|
|
prost_types::Value {
|
|
kind: Some(prost_types::value::Kind::BoolValue(true)),
|
|
},
|
|
))
|
|
.collect(),
|
|
},
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
ChainEntry {
|
|
name: "second".into(),
|
|
implementation: "test/two-stage".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
];
|
|
let described = runner
|
|
.describe_chain(&entries)
|
|
.await
|
|
.expect("describe two-stage chain");
|
|
let validator: Box<TransformedBodyValidator<'_>> =
|
|
Box::new(|_body: &[u8]| Err(miette!("OPA engine unavailable")));
|
|
|
|
let outcome = runner
|
|
.evaluate_described_with_policy(
|
|
&described,
|
|
input("original"),
|
|
TransformedBodyPolicy::Reevaluate(&*validator),
|
|
)
|
|
.await
|
|
.expect("policy evaluator failure should be a chain outcome");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert!(
|
|
outcome
|
|
.reason
|
|
.starts_with("transformed_body_policy_evaluation_failed:"),
|
|
"{}",
|
|
outcome.reason
|
|
);
|
|
assert_eq!(outcome.applied.len(), 1);
|
|
assert!(!second_ran.load(std::sync::atomic::Ordering::SeqCst));
|
|
}
|
|
|
|
fn scripted_service(result: openshell_core::proto::HttpRequestResult) -> ScriptedService {
|
|
ScriptedService {
|
|
manifest_name: BUILTIN_REGEX.into(),
|
|
max_body_bytes: 256 * 1024,
|
|
result,
|
|
}
|
|
}
|
|
|
|
fn allow_result() -> openshell_core::proto::HttpRequestResult {
|
|
openshell_core::proto::HttpRequestResult {
|
|
decision: Decision::Allow as i32,
|
|
reason: String::new(),
|
|
body: Vec::new(),
|
|
has_body: false,
|
|
header_mutations: Vec::new(),
|
|
findings: Vec::new(),
|
|
metadata: HashMap::new(),
|
|
reason_code: String::new(),
|
|
}
|
|
}
|
|
|
|
/// A middleware that records every evaluation it receives and allows the
|
|
/// request, for asserting what the supervisor actually sends to services.
|
|
struct RecordingService {
|
|
validated: std::sync::Mutex<Vec<ValidateConfigRequest>>,
|
|
received: std::sync::Mutex<Vec<HttpRequestEvaluation>>,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl SupervisorMiddleware for RecordingService {
|
|
type EvaluateWebSocketSessionStream = WebSocketResponseStream;
|
|
|
|
async fn evaluate_web_socket_session(
|
|
&self,
|
|
_request: Request<tonic::Streaming<openshell_core::proto::WebSocketSessionEvent>>,
|
|
) -> std::result::Result<tonic::Response<Self::EvaluateWebSocketSessionStream>, tonic::Status>
|
|
{
|
|
Err(tonic::Status::unimplemented("HTTP-only test middleware"))
|
|
}
|
|
|
|
async fn describe(
|
|
&self,
|
|
_request: Request<MiddlewareDescribeRequest>,
|
|
) -> std::result::Result<tonic::Response<MiddlewareManifest>, tonic::Status> {
|
|
Ok(tonic::Response::new(MiddlewareManifest {
|
|
name: "test/recorder".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
}))
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
request: Request<ValidateConfigRequest>,
|
|
) -> std::result::Result<tonic::Response<ValidateConfigResponse>, tonic::Status> {
|
|
self.validated
|
|
.lock()
|
|
.expect("validated config lock")
|
|
.push(request.into_inner());
|
|
Ok(tonic::Response::new(ValidateConfigResponse {
|
|
valid: true,
|
|
reason: String::new(),
|
|
}))
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: Request<HttpRequestEvaluation>,
|
|
) -> std::result::Result<
|
|
tonic::Response<openshell_core::proto::HttpRequestResult>,
|
|
tonic::Status,
|
|
> {
|
|
self.received
|
|
.lock()
|
|
.expect("recording lock")
|
|
.push(request.into_inner());
|
|
Ok(tonic::Response::new(allow_result()))
|
|
}
|
|
}
|
|
|
|
/// Three-stage service used to verify that each stage observes the header
|
|
/// state produced by all preceding stages.
|
|
struct HeaderChainService {
|
|
second_action: ExistingHeaderAction,
|
|
received: std::sync::Mutex<Vec<HttpRequestEvaluation>>,
|
|
}
|
|
|
|
struct InProcessHeaderChainService {
|
|
received: std::sync::Mutex<Vec<Vec<HttpHeader>>>,
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl InProcessMiddleware for InProcessHeaderChainService {
|
|
async fn describe(&self) -> MiddlewareManifest {
|
|
MiddlewareManifest {
|
|
name: "test/in-process-header-chain".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
}
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
_middleware_name: &str,
|
|
_config: &prost_types::Struct,
|
|
) -> Result<()> {
|
|
Ok(())
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: HttpRequestView<'_>,
|
|
) -> Result<openshell_core::proto::HttpRequestResult> {
|
|
let invocation = {
|
|
let mut received = self.received.lock().expect("in-process header chain lock");
|
|
let invocation = received.len();
|
|
received.push(request.headers().to_vec());
|
|
invocation
|
|
};
|
|
let mut result = allow_result();
|
|
if invocation == 0 {
|
|
result.header_mutations.push(write_header(
|
|
"cache-control",
|
|
"no-store",
|
|
ExistingHeaderAction::Overwrite,
|
|
));
|
|
}
|
|
Ok(result)
|
|
}
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl SupervisorMiddleware for HeaderChainService {
|
|
type EvaluateWebSocketSessionStream = WebSocketResponseStream;
|
|
|
|
async fn evaluate_web_socket_session(
|
|
&self,
|
|
_request: Request<tonic::Streaming<openshell_core::proto::WebSocketSessionEvent>>,
|
|
) -> std::result::Result<tonic::Response<Self::EvaluateWebSocketSessionStream>, tonic::Status>
|
|
{
|
|
Err(tonic::Status::unimplemented("HTTP-only test middleware"))
|
|
}
|
|
|
|
async fn describe(
|
|
&self,
|
|
_request: Request<MiddlewareDescribeRequest>,
|
|
) -> std::result::Result<tonic::Response<MiddlewareManifest>, tonic::Status> {
|
|
Ok(tonic::Response::new(MiddlewareManifest {
|
|
name: "test/header-chain".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpRequest as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
}))
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
_request: Request<ValidateConfigRequest>,
|
|
) -> std::result::Result<tonic::Response<ValidateConfigResponse>, tonic::Status> {
|
|
Ok(tonic::Response::new(ValidateConfigResponse {
|
|
valid: true,
|
|
reason: String::new(),
|
|
}))
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: Request<HttpRequestEvaluation>,
|
|
) -> std::result::Result<
|
|
tonic::Response<openshell_core::proto::HttpRequestResult>,
|
|
tonic::Status,
|
|
> {
|
|
let evaluation = request.into_inner();
|
|
let invocation = {
|
|
let mut received = self.received.lock().expect("header chain lock");
|
|
let invocation = received.len();
|
|
received.push(evaluation);
|
|
invocation
|
|
};
|
|
let mut result = allow_result();
|
|
if invocation == 0 {
|
|
result.header_mutations.push(write_header(
|
|
"cache-control",
|
|
"first",
|
|
ExistingHeaderAction::Overwrite,
|
|
));
|
|
} else if invocation == 1 {
|
|
result.header_mutations.push(write_header(
|
|
"cache-control",
|
|
"second",
|
|
self.second_action,
|
|
));
|
|
}
|
|
Ok(tonic::Response::new(result))
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn later_middleware_observes_prior_header_mutations() {
|
|
for (action, expected) in [
|
|
(ExistingHeaderAction::Append, vec!["first", "second"]),
|
|
(ExistingHeaderAction::Overwrite, vec!["second"]),
|
|
(ExistingHeaderAction::Skip, vec!["first"]),
|
|
] {
|
|
let service = Arc::new(HeaderChainService {
|
|
second_action: action,
|
|
received: std::sync::Mutex::new(Vec::new()),
|
|
});
|
|
let runner = ChainRunner::new_protobuf_for_tests(service.clone());
|
|
let entries = [
|
|
ChainEntry {
|
|
name: "first".into(),
|
|
implementation: "test/header-chain".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
ChainEntry {
|
|
name: "second".into(),
|
|
implementation: "test/header-chain".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
ChainEntry {
|
|
name: "observer".into(),
|
|
implementation: "test/header-chain".into(),
|
|
order: 20,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
];
|
|
|
|
let outcome = runner
|
|
.evaluate(&entries, input("payload"))
|
|
.await
|
|
.expect("evaluate header chain");
|
|
assert!(outcome.allowed);
|
|
let received = service.received.lock().expect("recorded header chain");
|
|
let observed: Vec<&str> = received[2]
|
|
.headers
|
|
.iter()
|
|
.filter(|header| header.name == "cache-control")
|
|
.map(|header| header.value.as_str())
|
|
.collect();
|
|
assert_eq!(observed, expected, "action {action:?}");
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn in_process_request_middleware_writes_end_to_end_header_without_namespace() {
|
|
let service = Arc::new(InProcessHeaderChainService {
|
|
received: std::sync::Mutex::new(Vec::new()),
|
|
});
|
|
let runner = ChainRunner::new(service.clone());
|
|
let entries = [
|
|
ChainEntry {
|
|
name: "writer".into(),
|
|
implementation: "test/in-process-header-chain".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
ChainEntry {
|
|
name: "observer".into(),
|
|
implementation: "test/in-process-header-chain".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
];
|
|
|
|
let outcome = runner
|
|
.evaluate(&entries, input("payload"))
|
|
.await
|
|
.expect("evaluate in-process header chain");
|
|
let received = service
|
|
.received
|
|
.lock()
|
|
.expect("recorded in-process headers");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(
|
|
received[1]
|
|
.iter()
|
|
.filter(|header| header.name == "cache-control")
|
|
.map(|header| header.value.as_str())
|
|
.collect::<Vec<_>>(),
|
|
vec!["no-store"]
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn repeated_request_headers_reach_middleware_in_wire_order() {
|
|
// A map contract would collapse repeated header names to one value
|
|
// while the upstream still receives every original value, creating an
|
|
// inspection differential. The service must see each entry in wire
|
|
// order.
|
|
let service = Arc::new(RecordingService {
|
|
validated: std::sync::Mutex::new(Vec::new()),
|
|
received: std::sync::Mutex::new(Vec::new()),
|
|
});
|
|
let recorder: Arc<MiddlewareService> = service.clone();
|
|
let runner = ChainRunner::new_protobuf_for_tests(recorder);
|
|
let validation_config = prost_types::Struct {
|
|
fields: std::iter::once((
|
|
"required".into(),
|
|
prost_types::Value {
|
|
kind: Some(prost_types::value::Kind::StringValue("present".into())),
|
|
},
|
|
))
|
|
.collect(),
|
|
};
|
|
runner
|
|
.validate_config("test/recorder", validation_config.clone())
|
|
.await
|
|
.expect("validate recorder config");
|
|
let evaluation_config = prost_types::Struct {
|
|
fields: std::iter::once((
|
|
"evaluation".into(),
|
|
prost_types::Value {
|
|
kind: Some(prost_types::value::Kind::StringValue("preserved".into())),
|
|
},
|
|
))
|
|
.collect(),
|
|
};
|
|
let recorder_entry = ChainEntry {
|
|
name: "recorder".into(),
|
|
implementation: "test/recorder".into(),
|
|
order: 0,
|
|
config: evaluation_config.clone(),
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
let mut request = input("payload");
|
|
request.headers = vec![
|
|
("x-api-key".into(), "first-value".into()),
|
|
("accept".into(), "application/json".into()),
|
|
("x-api-key".into(), "second-value".into()),
|
|
];
|
|
request.query = "page=2".into();
|
|
let original_body = request.body.as_ptr().addr();
|
|
|
|
let outcome = runner
|
|
.evaluate(&[recorder_entry], request)
|
|
.await
|
|
.expect("evaluate recording chain");
|
|
assert!(outcome.allowed);
|
|
|
|
let validated = service.validated.lock().expect("validated configs");
|
|
assert_eq!(validated.len(), 1);
|
|
assert_eq!(validated[0].middleware_name, "test/recorder");
|
|
assert_eq!(validated[0].config.as_ref(), Some(&validation_config));
|
|
drop(validated);
|
|
|
|
let received = service.received.lock().expect("recorded evaluations");
|
|
assert_eq!(received.len(), 1);
|
|
assert_eq!(outcome.body.as_ptr().addr(), original_body);
|
|
assert_ne!(received[0].body.as_ptr().addr(), original_body);
|
|
assert_eq!(received[0].body, b"payload");
|
|
assert_eq!(
|
|
received[0].phase,
|
|
SupervisorMiddlewarePhase::PreCredentials as i32
|
|
);
|
|
assert_eq!(received[0].middleware_name, "test/recorder");
|
|
assert_eq!(received[0].config.as_ref(), Some(&evaluation_config));
|
|
let context = received[0].context.as_ref().expect("request context");
|
|
assert_eq!(context.request_id, "req");
|
|
assert_eq!(context.sandbox_id, "sbx-id");
|
|
assert_eq!(context.sandbox, "sbx-name");
|
|
assert_eq!(context.workspace, "wrks-default");
|
|
assert!(context.originating_process.is_none());
|
|
let target = received[0].target.as_ref().expect("request target");
|
|
assert_eq!(target.scheme, "https");
|
|
assert_eq!(target.host, "api.example.com");
|
|
assert_eq!(target.port, 443);
|
|
assert_eq!(target.method, "POST");
|
|
assert_eq!(target.path, "/v1");
|
|
assert_eq!(target.query, "page=2");
|
|
let headers: Vec<(&str, &str)> = received[0]
|
|
.headers
|
|
.iter()
|
|
.map(|header| (header.name.as_str(), header.value.as_str()))
|
|
.collect();
|
|
assert_eq!(
|
|
headers,
|
|
vec![
|
|
("x-api-key", "first-value"),
|
|
("accept", "application/json"),
|
|
("x-api-key", "second-value"),
|
|
]
|
|
);
|
|
}
|
|
|
|
fn external_registration(max_payload_bytes: u64) -> SupervisorMiddlewareService {
|
|
SupervisorMiddlewareService {
|
|
name: "local-guard-service".into(),
|
|
grpc_endpoint: "http://127.0.0.1:50051".into(),
|
|
max_payload_bytes,
|
|
..Default::default()
|
|
}
|
|
}
|
|
|
|
async fn registry_with_external(
|
|
service: Arc<MiddlewareService>,
|
|
registration: SupervisorMiddlewareService,
|
|
) -> MiddlewareRegistry {
|
|
let builtin_service = services()
|
|
.into_iter()
|
|
.next()
|
|
.expect("built-in middleware service");
|
|
let builtin_manifest = builtin_service.describe().await;
|
|
validate_manifest_bindings("test built-in service", &builtin_manifest, None)
|
|
.expect("valid built-in manifest");
|
|
let builtin_name = builtin_manifest.name.clone();
|
|
let builtin_manifest_cell = OnceCell::new();
|
|
builtin_manifest_cell
|
|
.set(builtin_manifest)
|
|
.expect("built-in manifest cache");
|
|
|
|
let manifest = service
|
|
.describe(Request::new(MiddlewareDescribeRequest {
|
|
gateway: Some(gateway_metadata(ExtensionFamily::SupervisorMiddleware)),
|
|
}))
|
|
.await
|
|
.expect("describe test service")
|
|
.into_inner();
|
|
let operator_max_payload_bytes = usize::try_from(registration.max_payload_bytes).unwrap();
|
|
let operator_timeout = validate_registration(®istration).expect("valid registration");
|
|
validate_external_manifest(®istration, &manifest, operator_max_payload_bytes, false)
|
|
.expect("valid external manifest");
|
|
let manifest_cell = OnceCell::new();
|
|
manifest_cell.set(manifest).expect("manifest cache");
|
|
let registration_name = registration.name.clone();
|
|
MiddlewareRegistry {
|
|
services: Arc::new(vec![
|
|
Arc::new(MiddlewareServiceState {
|
|
attachment_name: Some(builtin_name.clone()),
|
|
service: MiddlewareDispatch::InProcess(builtin_service),
|
|
manifest: builtin_manifest_cell,
|
|
diagnostic_policy: MiddlewareDiagnosticPolicy::Preserve,
|
|
operator_max_payload_bytes: None,
|
|
operator_timeout: DEFAULT_MIDDLEWARE_TIMEOUT,
|
|
}),
|
|
Arc::new(MiddlewareServiceState {
|
|
attachment_name: Some(registration_name.clone()),
|
|
service: MiddlewareDispatch::Grpc(remote::GrpcMiddlewareService::from_service(
|
|
Arc::new(GeneratedMiddlewareEndpoint { service }),
|
|
)),
|
|
manifest: manifest_cell,
|
|
diagnostic_policy: MiddlewareDiagnosticPolicy::Normalize,
|
|
operator_max_payload_bytes: Some(operator_max_payload_bytes),
|
|
operator_timeout,
|
|
}),
|
|
]),
|
|
registered_services: Arc::new(vec![RegisteredMiddlewareService { registration }]),
|
|
middleware_names: Arc::new(HashSet::from([builtin_name, registration_name])),
|
|
negotiated_extensions: Arc::new(Vec::new()),
|
|
work_admission: Arc::new(Semaphore::new(MAX_CONCURRENT_MIDDLEWARE_WORK)),
|
|
work_admission_waiters: Arc::new(Semaphore::new(MAX_QUEUED_MIDDLEWARE_WORK)),
|
|
session_admission: Arc::new(Semaphore::new(MAX_CONCURRENT_MIDDLEWARE_SESSIONS)),
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn describe_chain_marks_resolved_and_unresolved_entries() {
|
|
let unresolved = ChainEntry {
|
|
name: "missing".into(),
|
|
implementation: "third-party/missing".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailOpen,
|
|
};
|
|
let described = builtin_runner()
|
|
.describe_chain(&[entry("redact", OnError::FailClosed), unresolved])
|
|
.await
|
|
.expect("describe chain");
|
|
// The built-in resolves and reports its real limit; the missing binding
|
|
// does not resolve and must not contribute a body limit.
|
|
assert!(described[0].is_resolved());
|
|
assert_eq!(described[0].max_payload_bytes(), 256 * 1024);
|
|
assert!(!described[1].is_resolved());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn descriptors_are_resolved_from_any_middleware_service() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: allow_result(),
|
|
}));
|
|
let entry = ChainEntry {
|
|
name: "external".into(),
|
|
implementation: "test/middleware".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
|
|
let described = runner
|
|
.describe_chain(std::slice::from_ref(&entry))
|
|
.await
|
|
.expect("describe external middleware");
|
|
assert_eq!(described[0].max_payload_bytes(), 4096);
|
|
assert_eq!(
|
|
described[0]
|
|
.binding
|
|
.as_ref()
|
|
.expect("described binding")
|
|
.phase,
|
|
SupervisorMiddlewarePhase::PreCredentials as i32
|
|
);
|
|
|
|
let outcome = runner
|
|
.evaluate_described(&described, input("hello"))
|
|
.await
|
|
.expect("evaluate external middleware");
|
|
assert!(outcome.allowed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn mixed_builtin_and_external_chain_uses_operator_limit() {
|
|
let external = Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: allow_result(),
|
|
});
|
|
let registry = registry_with_external(external, external_registration(1024)).await;
|
|
let runner = ChainRunner::from_registry(registry);
|
|
let external_entry = ChainEntry {
|
|
name: "external".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
let entries = [entry("builtin", OnError::FailClosed), external_entry];
|
|
|
|
let described = runner
|
|
.describe_chain(&entries)
|
|
.await
|
|
.expect("describe chain");
|
|
assert_eq!(described[0].max_payload_bytes(), 256 * 1024);
|
|
assert_eq!(described[1].max_payload_bytes(), 1024);
|
|
|
|
let outcome = runner
|
|
.evaluate_described(&described, input(r#"token="sk-ABCDEFGHIJKLMNOP""#))
|
|
.await
|
|
.expect("evaluate mixed chain");
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.applied.len(), 2);
|
|
assert_eq!(
|
|
String::from_utf8(outcome.body).expect("utf8"),
|
|
r#"token="[REDACTED]""#
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn undersized_stage_fails_open_while_later_stage_runs() {
|
|
// A body over one stage's limit must fail only that stage through its
|
|
// own `on_error`, not the whole chain: the 1 KiB fail-open guard is
|
|
// skipped while the 256 KiB fail-closed redactor still runs.
|
|
let external = Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: allow_result(),
|
|
});
|
|
let registry = registry_with_external(external, external_registration(1024)).await;
|
|
let runner = ChainRunner::from_registry(registry);
|
|
let guard_entry = ChainEntry {
|
|
name: "guard".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailOpen,
|
|
};
|
|
let mut redact_entry = entry("redact", OnError::FailClosed);
|
|
redact_entry.order = 10;
|
|
let entries = [guard_entry, redact_entry];
|
|
|
|
let body = format!("{}token=\"sk-ABCDEFGHIJKLMNOP\"", "x".repeat(1500));
|
|
let outcome = runner
|
|
.evaluate(&entries, input(&body))
|
|
.await
|
|
.expect("evaluate mixed-limit chain");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.applied.len(), 2);
|
|
assert!(
|
|
outcome.applied[0].failed,
|
|
"undersized guard must be skipped"
|
|
);
|
|
assert_eq!(outcome.applied[0].decision, Decision::Allow);
|
|
assert!(!outcome.applied[1].failed);
|
|
assert!(outcome.applied[1].transformed);
|
|
let body = String::from_utf8(outcome.body).expect("utf8");
|
|
assert!(body.contains("[REDACTED]"));
|
|
assert!(!body.contains("sk-ABCDEFGHIJKLMNOP"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn transformed_body_still_over_later_stage_capacity_honors_on_error() {
|
|
// Per-stage capacity applies to the current body: the redactor's
|
|
// replacement is still over the 1 KiB guard limit, so the fail-closed
|
|
// guard denies through its own `on_error` after the redactor ran.
|
|
let external = Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: allow_result(),
|
|
});
|
|
let registry = registry_with_external(external, external_registration(1024)).await;
|
|
let runner = ChainRunner::from_registry(registry);
|
|
let guard_entry = ChainEntry {
|
|
name: "guard".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
let entries = [entry("redact", OnError::FailClosed), guard_entry];
|
|
|
|
let body = format!("{}token=\"sk-ABCDEFGHIJKLMNOP\"", "x".repeat(1500));
|
|
let outcome = runner
|
|
.evaluate(&entries, input(&body))
|
|
.await
|
|
.expect("evaluate mixed-limit chain");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(
|
|
outcome.reason,
|
|
"middleware_failed: request_body_over_capacity"
|
|
);
|
|
assert_eq!(outcome.applied.len(), 2);
|
|
assert!(
|
|
outcome.applied[0].transformed,
|
|
"redactor ran before the deny"
|
|
);
|
|
assert!(outcome.applied[1].failed);
|
|
}
|
|
|
|
#[test]
|
|
fn external_manifest_rejects_operator_limit_above_capability() {
|
|
let registration = external_registration(4097);
|
|
let manifest = MiddlewareManifest {
|
|
name: "example/service".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: HTTP_REQUEST_OPERATION as i32,
|
|
phase: PRE_CREDENTIALS_PHASE as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
};
|
|
let error = validate_external_manifest(®istration, &manifest, 4097, false)
|
|
.expect_err("operator limit must fit capability");
|
|
assert!(error.to_string().contains("exceeds"));
|
|
}
|
|
|
|
#[test]
|
|
fn external_registration_rejects_payload_limit_above_platform_maximum() {
|
|
let registration = external_registration(u64::MAX);
|
|
let error = validate_registration(®istration)
|
|
.expect_err("extreme payload limit must be rejected before allocation");
|
|
assert!(error.to_string().contains("platform maximum"));
|
|
}
|
|
|
|
#[test]
|
|
fn manifest_rejects_payload_limit_above_platform_maximum() {
|
|
let registration = external_registration(4096);
|
|
let manifest = MiddlewareManifest {
|
|
name: "example/service".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: HTTP_REQUEST_OPERATION as i32,
|
|
phase: PRE_CREDENTIALS_PHASE as i32,
|
|
max_payload_bytes: u64::MAX,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
};
|
|
let error = validate_external_manifest(®istration, &manifest, 4096, false)
|
|
.expect_err("extreme advertised payload limit must be rejected");
|
|
assert!(error.to_string().contains("platform maximum"));
|
|
}
|
|
|
|
#[test]
|
|
fn manifest_rejects_duplicate_operation_phase_pairs() {
|
|
let registration = external_registration(4096);
|
|
let binding = || MiddlewareBinding {
|
|
operation: HTTP_REQUEST_OPERATION as i32,
|
|
phase: PRE_CREDENTIALS_PHASE as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: None,
|
|
};
|
|
let manifest = MiddlewareManifest {
|
|
name: "example/service".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![binding(), binding()],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
};
|
|
|
|
let error = validate_external_manifest(®istration, &manifest, 4096, false)
|
|
.expect_err("one service cannot advertise two bindings for the same pair");
|
|
assert!(
|
|
error
|
|
.to_string()
|
|
.contains("duplicate middleware operation/phase pair")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn manifest_accepts_http_response_pre_return_binding_when_dispatch_is_available() {
|
|
let registration = external_registration(4096);
|
|
let manifest = MiddlewareManifest {
|
|
name: "example/response".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::HttpResponse as i32,
|
|
phase: SupervisorMiddlewarePhase::PreReturn as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: Some(prost_types::Duration {
|
|
seconds: 0,
|
|
nanos: 500_000_000,
|
|
}),
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
};
|
|
|
|
validate_external_manifest(®istration, &manifest, 4096, false)
|
|
.expect("HTTP response pre-return binding is supported");
|
|
}
|
|
|
|
#[test]
|
|
fn manifest_accepts_forward_websocket_binding_and_reserves_return_phase() {
|
|
let binding = |phase| MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::WebsocketMessage as i32,
|
|
phase: phase as i32,
|
|
max_payload_bytes: MAX_MIDDLEWARE_PAYLOAD_BYTES as u64,
|
|
request_timeout: Some(proto_duration("500ms")),
|
|
};
|
|
let mut manifest = MiddlewareManifest {
|
|
name: "example/websocket".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![binding(SupervisorMiddlewarePhase::PreCredentials)],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
};
|
|
validate_manifest_bindings("test WebSocket service", &manifest, None)
|
|
.expect("forward WebSocket binding is supported");
|
|
|
|
manifest.bindings = vec![binding(SupervisorMiddlewarePhase::PreReturn)];
|
|
let error = validate_manifest_bindings("test WebSocket service", &manifest, None)
|
|
.expect_err("return-path WebSocket binding is not yet supported");
|
|
assert!(error.to_string().contains("not yet supported"));
|
|
}
|
|
|
|
#[test]
|
|
fn external_websocket_binding_requires_operator_payload_limit() {
|
|
let registration = external_registration(0);
|
|
let manifest = MiddlewareManifest {
|
|
name: "example/websocket".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::WebsocketMessage as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
};
|
|
|
|
let error = validate_external_manifest(®istration, &manifest, 0, false)
|
|
.expect_err("WebSocket bindings require an operator payload ceiling");
|
|
assert!(
|
|
error
|
|
.to_string()
|
|
.contains("must configure max_payload_bytes")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn external_websocket_binding_rejects_operator_limit_above_capability() {
|
|
let registration = external_registration(4097);
|
|
let manifest = MiddlewareManifest {
|
|
name: "example/websocket".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::WebsocketMessage as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: None,
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
};
|
|
|
|
let error = validate_external_manifest(®istration, &manifest, 4097, false)
|
|
.expect_err("operator payload limit must fit WebSocket capability");
|
|
assert!(error.to_string().contains("exceeds"));
|
|
}
|
|
|
|
#[test]
|
|
fn external_registration_accepts_http_and_https_grpc_endpoints() {
|
|
for grpc_endpoint in [
|
|
"http://127.0.0.1:50051",
|
|
"https://middleware.example.com:443",
|
|
] {
|
|
let mut registration = external_registration(4096);
|
|
registration.grpc_endpoint = grpc_endpoint.into();
|
|
validate_registration(®istration).expect("supported gRPC endpoint scheme");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn external_registration_rejects_unsupported_grpc_endpoint_scheme() {
|
|
let mut registration = external_registration(4096);
|
|
registration.grpc_endpoint = "ftp://middleware.example.com".into();
|
|
let error = validate_registration(®istration).expect_err("unsupported scheme");
|
|
assert!(error.to_string().contains("http:// or https://"));
|
|
}
|
|
|
|
#[test]
|
|
fn external_registration_name_is_stable_and_cannot_shadow_builtins() {
|
|
for name in ["", "guard\nforged", "openshell/regex"] {
|
|
let mut registration = external_registration(4096);
|
|
registration.name = name.into();
|
|
assert!(
|
|
validate_registration(®istration).is_err(),
|
|
"registration name {name:?} must be rejected"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn registration_timeout_uses_default_and_operator_override() {
|
|
let registration = external_registration(4096);
|
|
let timeout = validate_registration(®istration).expect("default timeout");
|
|
assert_eq!(timeout, DEFAULT_MIDDLEWARE_TIMEOUT);
|
|
|
|
let mut registration = external_registration(4096);
|
|
registration.request_timeout = Some(proto_duration("2s"));
|
|
let timeout = validate_registration(®istration).expect("operator timeout");
|
|
assert_eq!(timeout, Duration::from_secs(2));
|
|
}
|
|
|
|
#[test]
|
|
fn registration_timeout_enforces_bounds() {
|
|
for timeout in ["9ms", "31s"] {
|
|
let mut registration = external_registration(4096);
|
|
registration.request_timeout = Some(proto_duration(timeout));
|
|
assert!(validate_registration(®istration).is_err());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn manifest_binding_timeout_enforces_bounds() {
|
|
let registration = external_registration(4096);
|
|
for timeout in ["9ms", "31s"] {
|
|
let manifest = MiddlewareManifest {
|
|
name: "example/service".into(),
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: HTTP_REQUEST_OPERATION as i32,
|
|
phase: PRE_CREDENTIALS_PHASE as i32,
|
|
max_payload_bytes: 4096,
|
|
request_timeout: Some(proto_duration(timeout)),
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
};
|
|
let error = validate_external_manifest(®istration, &manifest, 4096, false)
|
|
.expect_err("out-of-bounds binding timeout must be rejected");
|
|
assert!(error.to_string().contains("invalid timeout"));
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn binding_timeout_override_controls_evaluation_and_on_error() {
|
|
let mut registration = external_registration(4096);
|
|
registration.request_timeout = Some(proto_duration("2s"));
|
|
let registry = registry_with_external(
|
|
Arc::new(SlowService {
|
|
delay: Duration::from_millis(50),
|
|
binding_timeout: Some(proto_duration("10ms")),
|
|
}),
|
|
registration,
|
|
)
|
|
.await;
|
|
let runner = ChainRunner::from_registry(registry);
|
|
let slow_entry = |on_error| ChainEntry {
|
|
name: "slow".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error,
|
|
};
|
|
|
|
let described = runner
|
|
.describe_chain(&[slow_entry(OnError::FailClosed)])
|
|
.await
|
|
.expect("describe slow binding");
|
|
assert_eq!(described[0].timeout(), Duration::from_millis(10));
|
|
|
|
let closed = runner
|
|
.evaluate(&[slow_entry(OnError::FailClosed)], input("payload"))
|
|
.await
|
|
.expect("fail-closed timeout outcome");
|
|
assert!(!closed.allowed);
|
|
assert_eq!(closed.reason, "middleware_failed: middleware_timeout");
|
|
|
|
let open = runner
|
|
.evaluate(&[slow_entry(OnError::FailOpen)], input("payload"))
|
|
.await
|
|
.expect("fail-open timeout outcome");
|
|
assert!(open.allowed);
|
|
assert!(open.applied[0].failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn operator_timeout_controls_binding_without_manifest_override() {
|
|
let mut registration = external_registration(4096);
|
|
registration.request_timeout = Some(proto_duration("10ms"));
|
|
let registry = registry_with_external(
|
|
Arc::new(SlowService {
|
|
delay: Duration::from_millis(50),
|
|
binding_timeout: None,
|
|
}),
|
|
registration,
|
|
)
|
|
.await;
|
|
let runner = ChainRunner::from_registry(registry);
|
|
let slow_entry = ChainEntry {
|
|
name: "slow".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
|
|
let described = runner
|
|
.describe_chain(std::slice::from_ref(&slow_entry))
|
|
.await
|
|
.expect("describe slow binding");
|
|
assert_eq!(described[0].timeout(), Duration::from_millis(10));
|
|
|
|
let outcome = runner
|
|
.evaluate(&[slow_entry], input("payload"))
|
|
.await
|
|
.expect("operator timeout outcome");
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(outcome.reason, "middleware_failed: middleware_timeout");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn operator_timeout_caps_longer_binding_timeout_for_validation_and_evaluation() {
|
|
let mut registration = external_registration(4096);
|
|
registration.request_timeout = Some(proto_duration("10ms"));
|
|
let registry = registry_with_external(
|
|
Arc::new(SlowService {
|
|
delay: Duration::from_millis(50),
|
|
binding_timeout: Some(prost_types::Duration {
|
|
seconds: 2,
|
|
nanos: 0,
|
|
}),
|
|
}),
|
|
registration,
|
|
)
|
|
.await;
|
|
let runner = ChainRunner::from_registry(registry);
|
|
let slow_entry = ChainEntry {
|
|
name: "slow".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
};
|
|
|
|
let described = runner
|
|
.describe_chain(std::slice::from_ref(&slow_entry))
|
|
.await
|
|
.expect("describe slow binding");
|
|
assert_eq!(described[0].timeout(), Duration::from_millis(10));
|
|
|
|
let validation_error = runner
|
|
.validate_config("local-guard-service", prost_types::Struct::default())
|
|
.await
|
|
.expect_err("operator timeout must cap ValidateConfig");
|
|
assert!(
|
|
validation_error
|
|
.to_string()
|
|
.contains("ValidateConfig failed")
|
|
);
|
|
assert!(validation_error.to_string().contains("timed out"));
|
|
|
|
let outcome = runner
|
|
.evaluate(&[slow_entry], input("payload"))
|
|
.await
|
|
.expect("operator-capped evaluation outcome");
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(outcome.reason, "middleware_failed: middleware_timeout");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn external_registry_attaches_same_service_under_multiple_names() {
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
.await
|
|
.expect("bind test middleware");
|
|
let address = listener.local_addr().expect("test middleware address");
|
|
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
|
|
let server = tonic::transport::Server::builder()
|
|
.add_service(SupervisorMiddlewareServer::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: allow_result(),
|
|
}))
|
|
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
|
|
let _ = shutdown_rx.await;
|
|
});
|
|
let server_task = tokio::spawn(server);
|
|
|
|
let mut registration = external_registration(1024);
|
|
registration.grpc_endpoint = format!("http://{address}");
|
|
let mut second_registration = registration.clone();
|
|
second_registration.name = "secondary-guard-service".into();
|
|
let registry = MiddlewareRegistry::connect_services(
|
|
Vec::new(),
|
|
vec![registration.clone(), second_registration.clone()],
|
|
)
|
|
.await
|
|
.expect("connect the same external middleware binding under two names");
|
|
let policy = SandboxPolicy {
|
|
network_middlewares: HashMap::from([(
|
|
"guard".into(),
|
|
NetworkMiddlewareConfig {
|
|
name: String::new(),
|
|
middleware: "local-guard-service".into(),
|
|
order: 0,
|
|
config: Some(prost_types::Struct::default()),
|
|
on_error: "fail_closed".into(),
|
|
endpoints: None,
|
|
},
|
|
)]),
|
|
..Default::default()
|
|
};
|
|
|
|
registry
|
|
.validate_policy_configs(&policy)
|
|
.await
|
|
.expect("remote config validates");
|
|
assert_eq!(
|
|
registry.required_services(Some(&policy)),
|
|
vec![registration.clone()]
|
|
);
|
|
|
|
let outcome = ChainRunner::from_registry(registry)
|
|
.evaluate(
|
|
&[
|
|
ChainEntry {
|
|
name: "primary".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
ChainEntry {
|
|
name: "secondary".into(),
|
|
implementation: "secondary-guard-service".into(),
|
|
order: 10,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
},
|
|
],
|
|
input("hello"),
|
|
)
|
|
.await
|
|
.expect("remote evaluation");
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.applied.len(), 2);
|
|
assert_eq!(outcome.applied[0].implementation, registration.name);
|
|
assert_eq!(outcome.applied[1].implementation, second_registration.name);
|
|
|
|
let _ = shutdown_tx.send(());
|
|
server_task
|
|
.await
|
|
.expect("join test middleware")
|
|
.expect("serve");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn remote_transport_accepts_maximum_bounded_request_and_response_envelopes() {
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
.await
|
|
.expect("bind test middleware");
|
|
let address = listener.local_addr().expect("test middleware address");
|
|
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
|
|
let response_findings = (0..MAX_MIDDLEWARE_FINDINGS_PER_STAGE)
|
|
.map(|_| Finding {
|
|
r#type: "f".repeat(1024),
|
|
label: "finding".into(),
|
|
count: 1,
|
|
confidence: "medium".into(),
|
|
severity: "medium".into(),
|
|
})
|
|
.collect();
|
|
let server = tonic::transport::Server::builder()
|
|
.add_service(
|
|
SupervisorMiddlewareServer::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: MAX_MIDDLEWARE_PAYLOAD_BYTES as u64,
|
|
result: openshell_core::proto::HttpRequestResult {
|
|
reason: "r".repeat(MAX_MIDDLEWARE_REASON_BYTES - 128),
|
|
reason_code: "r".repeat(MAX_MIDDLEWARE_REASON_CODE_BYTES),
|
|
body: vec![b'x'; MAX_MIDDLEWARE_PAYLOAD_BYTES],
|
|
has_body: true,
|
|
header_mutations: vec![write_header(
|
|
"x-openshell-middleware-envelope",
|
|
&"h".repeat(headers::MAX_HEADER_MUTATION_BYTES - 128),
|
|
ExistingHeaderAction::Append,
|
|
)],
|
|
findings: response_findings,
|
|
metadata: std::iter::once((
|
|
"diagnostic".into(),
|
|
"m".repeat(MAX_MIDDLEWARE_METADATA_BYTES - 128),
|
|
))
|
|
.collect(),
|
|
..allow_result()
|
|
},
|
|
})
|
|
.max_decoding_message_size(MIDDLEWARE_GRPC_MESSAGE_BYTES)
|
|
.max_encoding_message_size(MIDDLEWARE_GRPC_MESSAGE_BYTES),
|
|
)
|
|
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
|
|
let _ = shutdown_rx.await;
|
|
});
|
|
let server_task = tokio::spawn(server);
|
|
|
|
let mut registration = external_registration(MAX_MIDDLEWARE_PAYLOAD_BYTES as u64);
|
|
registration.grpc_endpoint = format!("http://{address}");
|
|
let registry = MiddlewareRegistry::connect_services(Vec::new(), vec![registration])
|
|
.await
|
|
.expect("connect external middleware");
|
|
let config = prost_types::Struct {
|
|
fields: std::iter::once((
|
|
"payload".into(),
|
|
prost_types::Value {
|
|
kind: Some(prost_types::value::Kind::StringValue(
|
|
"c".repeat(MAX_MIDDLEWARE_CONFIG_BYTES - 256),
|
|
)),
|
|
},
|
|
))
|
|
.collect(),
|
|
};
|
|
assert!(config.encoded_len() <= MAX_MIDDLEWARE_CONFIG_BYTES);
|
|
let mut request = input("");
|
|
request.request_id = "r".repeat(MAX_MIDDLEWARE_CONTEXT_BYTES - 256);
|
|
request.path = format!("/{}", "p".repeat(MAX_MIDDLEWARE_TARGET_BYTES - 512));
|
|
request.headers = vec![(
|
|
"x-large-envelope".into(),
|
|
"v".repeat(MAX_MIDDLEWARE_HEADER_BYTES - 256),
|
|
)];
|
|
request.body = vec![b'b'; MAX_MIDDLEWARE_PAYLOAD_BYTES];
|
|
let outcome = ChainRunner::from_registry(registry)
|
|
.evaluate(
|
|
&[ChainEntry {
|
|
name: "guard".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config,
|
|
on_error: OnError::FailClosed,
|
|
}],
|
|
request,
|
|
)
|
|
.await
|
|
.expect("maximum bounded envelopes should fit configured transport limit");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.body.len(), MAX_MIDDLEWARE_PAYLOAD_BYTES);
|
|
assert_eq!(outcome.header_mutations.len(), 1);
|
|
assert_eq!(outcome.findings.len(), MAX_MIDDLEWARE_FINDINGS_PER_STAGE);
|
|
let _ = shutdown_tx.send(());
|
|
server_task
|
|
.await
|
|
.expect("join test middleware")
|
|
.expect("serve");
|
|
}
|
|
|
|
#[test]
|
|
fn grpc_envelope_headroom_matches_bounded_components() {
|
|
assert_eq!(MIDDLEWARE_GRPC_ENVELOPE_BYTES, 292 * 1024 + 64);
|
|
assert_eq!(
|
|
MIDDLEWARE_GRPC_MESSAGE_BYTES,
|
|
MAX_MIDDLEWARE_PAYLOAD_BYTES + 292 * 1024 + 64
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn external_diagnostics_are_normalized_before_reaching_logs() {
|
|
let secret = "sk-secret-request-value";
|
|
let registration = external_registration(4096);
|
|
let service = Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: openshell_core::proto::HttpRequestResult {
|
|
decision: Decision::Deny as i32,
|
|
reason: format!("denied body={secret}\nFINDING:FORGED"),
|
|
reason_code: "content_match".into(),
|
|
findings: vec![Finding {
|
|
r#type: format!("secret.{secret}\nforged"),
|
|
label: format!("matched {secret}\nFINDING:FORGED"),
|
|
count: 1,
|
|
confidence: secret.into(),
|
|
severity: "high\nFINDING:FORGED".into(),
|
|
}],
|
|
metadata: std::iter::once(("request".into(), secret.into())).collect(),
|
|
..allow_result()
|
|
},
|
|
});
|
|
let registry = registry_with_external(service, registration).await;
|
|
let outcome = ChainRunner::from_registry(registry)
|
|
.evaluate(
|
|
&[ChainEntry {
|
|
name: "guard".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
}],
|
|
input("hello"),
|
|
)
|
|
.await
|
|
.expect("evaluate external middleware");
|
|
|
|
assert_eq!(outcome.reason, "middleware_denied:guard:content_match");
|
|
assert_eq!(
|
|
outcome.denial,
|
|
Some(MiddlewareDenial {
|
|
config_name: "guard".into(),
|
|
reason_code: Some("content_match".into()),
|
|
})
|
|
);
|
|
assert_eq!(
|
|
outcome.findings[0].finding.r#type,
|
|
"local-guard-service.finding"
|
|
);
|
|
assert_eq!(outcome.findings[0].finding.label, EXTERNAL_FINDING_LABEL);
|
|
assert_eq!(outcome.findings[0].finding.severity, "medium");
|
|
assert!(outcome.metadata.is_empty());
|
|
assert!(!format!("{outcome:?}").contains(secret));
|
|
assert!(!format!("{outcome:?}").contains("FINDING:FORGED"));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn invalid_reason_code_is_a_middleware_failure() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(scripted_service(
|
|
openshell_core::proto::HttpRequestResult {
|
|
decision: Decision::Deny as i32,
|
|
reason_code: "Secret value!".into(),
|
|
..allow_result()
|
|
},
|
|
)));
|
|
let outcome = runner
|
|
.evaluate(
|
|
&[entry("content-guard", OnError::FailClosed)],
|
|
input("hello"),
|
|
)
|
|
.await
|
|
.expect("evaluate invalid reason code");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(
|
|
outcome.reason,
|
|
"middleware_failed: response_reason_code_invalid"
|
|
);
|
|
assert!(outcome.denial.is_none());
|
|
assert!(outcome.applied[0].failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn external_header_mutation_failure_uses_platform_reason() {
|
|
let secret = "sk-secret-request-value";
|
|
let registration = external_registration(4096);
|
|
let service = Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: openshell_core::proto::HttpRequestResult {
|
|
header_mutations: vec![write_header(
|
|
&format!("x-openshell-middleware-invalid\n{secret}"),
|
|
"value",
|
|
ExistingHeaderAction::Append,
|
|
)],
|
|
..allow_result()
|
|
},
|
|
});
|
|
let registry = registry_with_external(service, registration).await;
|
|
let outcome = ChainRunner::from_registry(registry)
|
|
.evaluate(
|
|
&[ChainEntry {
|
|
name: "guard".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
}],
|
|
input("hello"),
|
|
)
|
|
.await
|
|
.expect("evaluate external middleware");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(
|
|
outcome.reason,
|
|
"middleware_failed: header_mutation_invalid_name"
|
|
);
|
|
assert!(outcome.findings.is_empty());
|
|
assert!(!format!("{outcome:?}").contains(secret));
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn credential_placeholder_header_mutation_follows_on_error() {
|
|
let placeholder = "openshell:resolve:env:API_KEY";
|
|
let service = Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: openshell_core::proto::HttpRequestResult {
|
|
header_mutations: vec![write_header(
|
|
"x-api-key",
|
|
placeholder,
|
|
ExistingHeaderAction::Overwrite,
|
|
)],
|
|
..allow_result()
|
|
},
|
|
});
|
|
let registry = registry_with_external(service, external_registration(4096)).await;
|
|
let runner = ChainRunner::from_registry(registry);
|
|
|
|
for (on_error, allowed) in [(OnError::FailClosed, false), (OnError::FailOpen, true)] {
|
|
let outcome = runner
|
|
.evaluate(
|
|
&[ChainEntry {
|
|
name: "guard".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error,
|
|
}],
|
|
input("hello"),
|
|
)
|
|
.await
|
|
.expect("evaluate credential placeholder mutation");
|
|
|
|
assert_eq!(outcome.allowed, allowed);
|
|
assert!(outcome.header_mutations.is_empty());
|
|
assert_eq!(outcome.applied.len(), 1);
|
|
assert!(outcome.applied[0].failed);
|
|
assert!(!format!("{outcome:?}").contains(placeholder));
|
|
if !allowed {
|
|
assert_eq!(
|
|
outcome.reason,
|
|
"middleware_failed: header_mutation_credential_placeholder"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn connection_nominated_write_and_remove_are_rejected_after_filtering() {
|
|
let mutations = [
|
|
write_header(
|
|
"x-openshell-middleware-tag",
|
|
"value",
|
|
ExistingHeaderAction::Append,
|
|
),
|
|
HeaderMutation {
|
|
operation: Some(header_mutation::Operation::Remove(
|
|
openshell_core::proto::RemoveHeader {
|
|
name: "x-openshell-middleware-tag".into(),
|
|
},
|
|
)),
|
|
},
|
|
];
|
|
|
|
for mutation in mutations {
|
|
let service = Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: openshell_core::proto::HttpRequestResult {
|
|
header_mutations: vec![mutation],
|
|
..allow_result()
|
|
},
|
|
});
|
|
let registry = registry_with_external(service, external_registration(4096)).await;
|
|
let mut request = input("hello");
|
|
request.connection_nominated_headers = vec!["x-openshell-middleware-tag".into()];
|
|
|
|
let outcome = ChainRunner::from_registry(registry)
|
|
.evaluate(
|
|
&[ChainEntry {
|
|
name: "guard".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
}],
|
|
request,
|
|
)
|
|
.await
|
|
.expect("evaluate external middleware");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(
|
|
outcome.reason,
|
|
"middleware_failed: header_mutation_hop_by_hop_header"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn finding_overflow_is_an_invalid_response_governed_by_on_error() {
|
|
let registration = external_registration(4096);
|
|
let service = Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: openshell_core::proto::HttpRequestResult {
|
|
findings: vec![Finding::default(); MAX_MIDDLEWARE_FINDINGS_PER_STAGE + 1],
|
|
..allow_result()
|
|
},
|
|
});
|
|
let registry = registry_with_external(service, registration).await;
|
|
let runner = ChainRunner::from_registry(registry);
|
|
|
|
for (on_error, allowed) in [(OnError::FailClosed, false), (OnError::FailOpen, true)] {
|
|
let outcome = runner
|
|
.evaluate(
|
|
&[ChainEntry {
|
|
name: "guard".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error,
|
|
}],
|
|
input("hello"),
|
|
)
|
|
.await
|
|
.expect("evaluate finding overflow");
|
|
|
|
assert_eq!(outcome.allowed, allowed);
|
|
assert!(outcome.findings.is_empty());
|
|
assert_eq!(outcome.applied.len(), 1);
|
|
assert!(outcome.applied[0].failed);
|
|
if !allowed {
|
|
assert_eq!(
|
|
outcome.reason,
|
|
"middleware_failed: response_findings_over_capacity"
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn maximum_chain_retains_findings_from_every_stage() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(ScriptedService {
|
|
manifest_name: "test/middleware".into(),
|
|
max_body_bytes: 4096,
|
|
result: openshell_core::proto::HttpRequestResult {
|
|
findings: vec![
|
|
Finding {
|
|
r#type: "example.finding".into(),
|
|
label: "Example finding".into(),
|
|
count: 1,
|
|
confidence: String::new(),
|
|
severity: "medium".into(),
|
|
};
|
|
MAX_MIDDLEWARE_FINDINGS_PER_STAGE
|
|
],
|
|
..allow_result()
|
|
},
|
|
}));
|
|
let entries: Vec<_> = (0..MAX_MIDDLEWARE_CHAIN_STAGES)
|
|
.map(|index| ChainEntry {
|
|
name: format!("guard-{index}"),
|
|
implementation: "test/middleware".into(),
|
|
order: i32::try_from(index).expect("bounded stage index"),
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
})
|
|
.collect();
|
|
|
|
let outcome = runner
|
|
.evaluate(&entries, input("hello"))
|
|
.await
|
|
.expect("evaluate maximum chain");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.applied.len(), MAX_MIDDLEWARE_CHAIN_STAGES);
|
|
assert_eq!(outcome.findings.len(), MAX_MIDDLEWARE_CHAIN_FINDINGS);
|
|
for (stage, findings) in outcome
|
|
.findings
|
|
.chunks_exact(MAX_MIDDLEWARE_FINDINGS_PER_STAGE)
|
|
.enumerate()
|
|
{
|
|
assert!(
|
|
findings
|
|
.iter()
|
|
.all(|finding| finding.middleware == format!("guard-{stage}"))
|
|
);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn deny_decision_short_circuits_chain() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(scripted_service(
|
|
openshell_core::proto::HttpRequestResult {
|
|
decision: Decision::Deny as i32,
|
|
reason: "blocked_by_policy".into(),
|
|
..allow_result()
|
|
},
|
|
)));
|
|
let outcome = runner
|
|
.evaluate(
|
|
&[
|
|
entry("first", OnError::FailClosed),
|
|
entry("second", OnError::FailClosed),
|
|
],
|
|
input("hello"),
|
|
)
|
|
.await
|
|
.expect("evaluate");
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(outcome.reason, "middleware_denied:first");
|
|
assert_eq!(
|
|
outcome.denial,
|
|
Some(MiddlewareDenial {
|
|
config_name: "first".into(),
|
|
reason_code: None,
|
|
})
|
|
);
|
|
assert!(!format!("{outcome:?}").contains("blocked_by_policy"));
|
|
// The deny short-circuits the chain: the second middleware never runs.
|
|
assert_eq!(outcome.applied.len(), 1);
|
|
assert_eq!(outcome.applied[0].decision, Decision::Deny);
|
|
assert!(!outcome.applied[0].failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn deny_decision_ignores_unsafe_mutations_under_fail_open() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(scripted_service(
|
|
openshell_core::proto::HttpRequestResult {
|
|
decision: Decision::Deny as i32,
|
|
reason: "blocked_by_policy".into(),
|
|
header_mutations: vec![write_header(
|
|
"x-openshell-middleware-inject",
|
|
"ok\r\nHost: evil",
|
|
ExistingHeaderAction::Append,
|
|
)],
|
|
..allow_result()
|
|
},
|
|
)));
|
|
|
|
let outcome = runner
|
|
.evaluate(&[entry("guard", OnError::FailOpen)], input("hello"))
|
|
.await
|
|
.expect("evaluate");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(outcome.reason, "middleware_denied:guard");
|
|
assert!(outcome.header_mutations.is_empty());
|
|
assert_eq!(outcome.applied.len(), 1);
|
|
assert_eq!(outcome.applied[0].decision, Decision::Deny);
|
|
assert!(!outcome.applied[0].failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn deny_decision_ignores_oversized_replacement_under_fail_open() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(ScriptedService {
|
|
manifest_name: BUILTIN_REGEX.into(),
|
|
max_body_bytes: 4,
|
|
result: openshell_core::proto::HttpRequestResult {
|
|
decision: Decision::Deny as i32,
|
|
reason: "blocked_by_policy".into(),
|
|
body: b"too large".to_vec(),
|
|
has_body: true,
|
|
..allow_result()
|
|
},
|
|
}));
|
|
|
|
let outcome = runner
|
|
.evaluate(&[entry("guard", OnError::FailOpen)], input("safe"))
|
|
.await
|
|
.expect("evaluate");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(outcome.reason, "middleware_denied:guard");
|
|
assert_eq!(outcome.body, b"safe");
|
|
assert_eq!(outcome.applied.len(), 1);
|
|
assert_eq!(outcome.applied[0].decision, Decision::Deny);
|
|
assert!(!outcome.applied[0].transformed);
|
|
assert!(!outcome.applied[0].failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn metadata_and_findings_are_namespaced_per_config() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(scripted_service(
|
|
openshell_core::proto::HttpRequestResult {
|
|
findings: vec![Finding {
|
|
r#type: "pii.email".into(),
|
|
label: "email address".into(),
|
|
count: 2,
|
|
confidence: "high".into(),
|
|
severity: "medium".into(),
|
|
}],
|
|
metadata: std::iter::once(("sensitivity".to_string(), "high".to_string()))
|
|
.collect(),
|
|
..allow_result()
|
|
},
|
|
)));
|
|
let outcome = runner
|
|
.evaluate(
|
|
&[
|
|
entry("alpha", OnError::FailClosed),
|
|
entry("beta", OnError::FailClosed),
|
|
],
|
|
input("hello"),
|
|
)
|
|
.await
|
|
.expect("evaluate");
|
|
assert!(outcome.allowed);
|
|
// Metadata is bucketed under each config's local name, so two configs
|
|
// emitting the same key do not collide.
|
|
assert_eq!(outcome.metadata["alpha"]["sensitivity"], "high");
|
|
assert_eq!(outcome.metadata["beta"]["sensitivity"], "high");
|
|
// Findings are tagged with the emitting config's name.
|
|
assert_eq!(outcome.findings.len(), 2);
|
|
assert_eq!(outcome.findings[0].middleware, "alpha");
|
|
assert_eq!(outcome.findings[1].middleware, "beta");
|
|
assert_eq!(outcome.findings[0].finding.r#type, "pii.email");
|
|
assert_eq!(outcome.findings[0].finding.count, 2);
|
|
}
|
|
|
|
fn unsafe_header_service() -> ScriptedService {
|
|
scripted_service(openshell_core::proto::HttpRequestResult {
|
|
header_mutations: vec![
|
|
write_header(
|
|
"x-openshell-middleware-safe",
|
|
"safe",
|
|
ExistingHeaderAction::Append,
|
|
),
|
|
write_header(
|
|
"x-openshell-middleware-inject",
|
|
"ok\r\nHost: evil",
|
|
ExistingHeaderAction::Append,
|
|
),
|
|
],
|
|
..allow_result()
|
|
})
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn malformed_response_headers_fail_closed_denies() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(unsafe_header_service()));
|
|
let outcome = runner
|
|
.evaluate(&[entry("redact", OnError::FailClosed)], input("hello"))
|
|
.await
|
|
.expect("evaluate");
|
|
assert!(!outcome.allowed);
|
|
assert!(outcome.reason.starts_with("middleware_failed:"));
|
|
// The deny reason names the offending header so operators can fix the
|
|
// service without reading supervisor source.
|
|
assert!(
|
|
outcome.reason.contains("x-openshell-middleware-inject"),
|
|
"reason should name the offending header: {}",
|
|
outcome.reason
|
|
);
|
|
assert!(outcome.applied.iter().any(|inv| inv.failed));
|
|
// The stage is atomic: neither the unsafe mutation nor the safe
|
|
// mutation preceding it is forwarded.
|
|
assert!(outcome.header_mutations.is_empty());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn malformed_response_headers_fail_open_continues() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(unsafe_header_service()));
|
|
let outcome = runner
|
|
.evaluate(&[entry("redact", OnError::FailOpen)], input("hello"))
|
|
.await
|
|
.expect("evaluate");
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.body, b"hello");
|
|
assert!(outcome.header_mutations.is_empty());
|
|
assert_eq!(outcome.applied.len(), 1);
|
|
assert!(outcome.applied[0].failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn oversized_replacement_body_honors_on_error() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(ScriptedService {
|
|
manifest_name: BUILTIN_REGEX.into(),
|
|
max_body_bytes: 4,
|
|
result: openshell_core::proto::HttpRequestResult {
|
|
body: b"too large".to_vec(),
|
|
has_body: true,
|
|
..allow_result()
|
|
},
|
|
}));
|
|
let fail_open = entry("small", OnError::FailOpen);
|
|
let mut fail_closed = fail_open.clone();
|
|
fail_closed.on_error = OnError::FailClosed;
|
|
|
|
let open_outcome = runner
|
|
.evaluate(&[fail_open], input("safe"))
|
|
.await
|
|
.expect("fail-open evaluation");
|
|
assert!(open_outcome.allowed);
|
|
assert_eq!(open_outcome.body, b"safe");
|
|
assert!(open_outcome.applied[0].failed);
|
|
|
|
let closed_outcome = runner
|
|
.evaluate(&[fail_closed], input("safe"))
|
|
.await
|
|
.expect("fail-closed evaluation");
|
|
assert!(!closed_outcome.allowed);
|
|
assert_eq!(
|
|
closed_outcome.reason,
|
|
"middleware_failed: response_body_over_capacity"
|
|
);
|
|
assert!(closed_outcome.applied[0].failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn oversized_request_body_honors_on_error() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(ScriptedService {
|
|
manifest_name: BUILTIN_REGEX.into(),
|
|
max_body_bytes: 4,
|
|
result: allow_result(),
|
|
}));
|
|
let fail_open = entry("small", OnError::FailOpen);
|
|
let mut fail_closed = fail_open.clone();
|
|
fail_closed.on_error = OnError::FailClosed;
|
|
|
|
let open_outcome = runner
|
|
.evaluate(&[fail_open], input("hello"))
|
|
.await
|
|
.expect("fail-open evaluation");
|
|
assert!(open_outcome.allowed);
|
|
assert_eq!(open_outcome.body, b"hello");
|
|
assert!(open_outcome.applied[0].failed);
|
|
|
|
let closed_outcome = runner
|
|
.evaluate(&[fail_closed], input("hello"))
|
|
.await
|
|
.expect("fail-closed evaluation");
|
|
assert!(!closed_outcome.allowed);
|
|
assert_eq!(
|
|
closed_outcome.reason,
|
|
"middleware_failed: request_body_over_capacity"
|
|
);
|
|
assert!(closed_outcome.applied[0].failed);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unspecified_decision_uses_fail_closed() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(scripted_service(
|
|
openshell_core::proto::HttpRequestResult {
|
|
decision: Decision::Unspecified as i32,
|
|
..allow_result()
|
|
},
|
|
)));
|
|
|
|
let outcome = runner
|
|
.evaluate(&[entry("redact", OnError::FailClosed)], input("hello"))
|
|
.await
|
|
.expect("evaluate");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(
|
|
outcome.reason,
|
|
"middleware_failed: invalid_response_decision"
|
|
);
|
|
assert!(outcome.applied[0].failed);
|
|
}
|
|
|
|
#[derive(Clone, Default)]
|
|
struct OpenAiRedactionService {
|
|
preflight: Arc<std::sync::Mutex<Option<openshell_core::proto::WebSocketPreflight>>>,
|
|
manifest_name: String,
|
|
describe_calls: Arc<std::sync::atomic::AtomicUsize>,
|
|
skip: bool,
|
|
deny: bool,
|
|
preflight_reason: String,
|
|
preflight_reason_code: String,
|
|
preflight_findings: Vec<Finding>,
|
|
preflight_metadata: HashMap<String, String>,
|
|
close_on_first_message: bool,
|
|
messages: Arc<std::sync::atomic::AtomicUsize>,
|
|
session_ends: Option<
|
|
tokio::sync::mpsc::UnboundedSender<openshell_core::proto::MiddlewareSessionEndReason>,
|
|
>,
|
|
}
|
|
|
|
impl OpenAiRedactionService {
|
|
fn websocket_stream<S>(&self, mut requests: S) -> WebSocketResponseStream
|
|
where
|
|
S: Stream<
|
|
Item = std::result::Result<
|
|
openshell_core::proto::WebSocketSessionEvent,
|
|
tonic::Status,
|
|
>,
|
|
> + Send
|
|
+ Unpin
|
|
+ 'static,
|
|
{
|
|
use openshell_core::proto::{
|
|
WebSocketMessageResult, WebSocketPreflightAction, WebSocketPreflightDecision,
|
|
WebSocketSessionEventResult, web_socket_message, web_socket_message_result,
|
|
web_socket_session_event, web_socket_session_event_result,
|
|
};
|
|
|
|
let preflight = Arc::clone(&self.preflight);
|
|
let skip = self.skip;
|
|
let deny = self.deny;
|
|
let preflight_reason = self.preflight_reason.clone();
|
|
let preflight_reason_code = self.preflight_reason_code.clone();
|
|
let preflight_findings = self.preflight_findings.clone();
|
|
let preflight_metadata = self.preflight_metadata.clone();
|
|
let close_on_first_message = self.close_on_first_message;
|
|
let messages = Arc::clone(&self.messages);
|
|
let session_ends = self.session_ends.clone();
|
|
let (responses_tx, responses_rx) = tokio::sync::mpsc::channel(4);
|
|
tokio::spawn(async move {
|
|
while let Some(Ok(request)) = requests.next().await {
|
|
let response = match request.event {
|
|
Some(web_socket_session_event::Event::Preflight(value)) => {
|
|
*preflight.lock().expect("preflight lock") = Some(value);
|
|
Some(WebSocketSessionEventResult {
|
|
result: Some(
|
|
web_socket_session_event_result::Result::PreflightDecision(
|
|
WebSocketPreflightDecision {
|
|
action: if deny {
|
|
WebSocketPreflightAction::Deny as i32
|
|
} else if skip {
|
|
WebSocketPreflightAction::Skip as i32
|
|
} else {
|
|
WebSocketPreflightAction::Inspect as i32
|
|
},
|
|
reason: preflight_reason.clone(),
|
|
findings: preflight_findings.clone(),
|
|
metadata: preflight_metadata.clone(),
|
|
reason_code: preflight_reason_code.clone(),
|
|
},
|
|
),
|
|
),
|
|
})
|
|
}
|
|
Some(web_socket_session_event::Event::Message(value)) => {
|
|
messages.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
|
if close_on_first_message {
|
|
break;
|
|
}
|
|
let web_socket_message::Payload::Text(payload) =
|
|
value.payload.expect("test OpenAI event payload")
|
|
else {
|
|
panic!("test OpenAI event must be text");
|
|
};
|
|
let payload = payload.replace("customer-secret", "[REDACTED]");
|
|
Some(WebSocketSessionEventResult {
|
|
result: Some(
|
|
web_socket_session_event_result::Result::MessageResult(
|
|
WebSocketMessageResult {
|
|
sequence: value.sequence,
|
|
decision: Decision::Allow as i32,
|
|
replacement: Some(
|
|
web_socket_message_result::Replacement::Text(
|
|
payload,
|
|
),
|
|
),
|
|
reason_code: "redacted".into(),
|
|
..Default::default()
|
|
},
|
|
),
|
|
),
|
|
})
|
|
}
|
|
Some(web_socket_session_event::Event::SessionStart(_)) | None => None,
|
|
Some(web_socket_session_event::Event::SessionEnd(end)) => {
|
|
if let Some(session_ends) = &session_ends
|
|
&& let Ok(reason) =
|
|
openshell_core::proto::MiddlewareSessionEndReason::try_from(
|
|
end.reason,
|
|
)
|
|
{
|
|
let _ = session_ends.send(reason);
|
|
}
|
|
None
|
|
}
|
|
};
|
|
if let Some(response) = response
|
|
&& responses_tx.send(Ok(response)).await.is_err()
|
|
{
|
|
break;
|
|
}
|
|
}
|
|
});
|
|
Box::pin(tokio_stream::wrappers::ReceiverStream::new(responses_rx))
|
|
}
|
|
}
|
|
|
|
fn websocket_preflight_input(session_id: impl Into<String>) -> WebSocketPreflightInput {
|
|
WebSocketPreflightInput {
|
|
session_id: session_id.into(),
|
|
request_id: "request".into(),
|
|
sandbox_id: "sandbox".into(),
|
|
sandbox_name: "sandbox-name".into(),
|
|
workspace: "wrks-default".into(),
|
|
scheme: "wss".into(),
|
|
host: "api.openai.com".into(),
|
|
port: 443,
|
|
path: "/v1/responses".into(),
|
|
requested_subprotocols: Vec::new(),
|
|
}
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl SupervisorMiddleware for OpenAiRedactionService {
|
|
type EvaluateWebSocketSessionStream = WebSocketResponseStream;
|
|
|
|
async fn describe(
|
|
&self,
|
|
_request: Request<MiddlewareDescribeRequest>,
|
|
) -> std::result::Result<tonic::Response<MiddlewareManifest>, tonic::Status> {
|
|
self.describe_calls
|
|
.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
|
Ok(tonic::Response::new(MiddlewareManifest {
|
|
name: if self.manifest_name.is_empty() {
|
|
"test/openai-websocket-redactor".into()
|
|
} else {
|
|
self.manifest_name.clone()
|
|
},
|
|
service_version: "test".into(),
|
|
bindings: vec![MiddlewareBinding {
|
|
operation: SupervisorMiddlewareOperation::WebsocketMessage as i32,
|
|
phase: SupervisorMiddlewarePhase::PreCredentials as i32,
|
|
max_payload_bytes: MAX_MIDDLEWARE_PAYLOAD_BYTES as u64,
|
|
request_timeout: Some(prost_types::Duration {
|
|
seconds: 1,
|
|
nanos: 0,
|
|
}),
|
|
}],
|
|
expected_audience: String::new(),
|
|
extension: Some(openshell_core::extension_protocol::extension_metadata(
|
|
ExtensionFamily::SupervisorMiddleware,
|
|
"openshell/test-middleware",
|
|
"test",
|
|
[],
|
|
)),
|
|
}))
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
_request: Request<ValidateConfigRequest>,
|
|
) -> std::result::Result<tonic::Response<ValidateConfigResponse>, tonic::Status> {
|
|
Ok(tonic::Response::new(ValidateConfigResponse {
|
|
valid: true,
|
|
reason: String::new(),
|
|
}))
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
_request: Request<HttpRequestEvaluation>,
|
|
) -> std::result::Result<
|
|
tonic::Response<openshell_core::proto::HttpRequestResult>,
|
|
tonic::Status,
|
|
> {
|
|
Err(tonic::Status::unimplemented(
|
|
"WebSocket-only test middleware",
|
|
))
|
|
}
|
|
|
|
async fn evaluate_web_socket_session(
|
|
&self,
|
|
request: Request<tonic::Streaming<openshell_core::proto::WebSocketSessionEvent>>,
|
|
) -> std::result::Result<tonic::Response<Self::EvaluateWebSocketSessionStream>, tonic::Status>
|
|
{
|
|
Ok(tonic::Response::new(
|
|
self.websocket_stream(request.into_inner()),
|
|
))
|
|
}
|
|
}
|
|
|
|
#[tonic::async_trait]
|
|
impl SupervisorMiddlewareEndpoint for OpenAiRedactionService {
|
|
async fn describe(
|
|
&self,
|
|
request: Request<MiddlewareDescribeRequest>,
|
|
) -> std::result::Result<tonic::Response<MiddlewareManifest>, tonic::Status> {
|
|
SupervisorMiddleware::describe(self, request).await
|
|
}
|
|
|
|
async fn validate_config(
|
|
&self,
|
|
request: Request<ValidateConfigRequest>,
|
|
) -> std::result::Result<tonic::Response<ValidateConfigResponse>, tonic::Status> {
|
|
SupervisorMiddleware::validate_config(self, request).await
|
|
}
|
|
|
|
async fn evaluate_http_request(
|
|
&self,
|
|
request: Request<HttpRequestEvaluation>,
|
|
) -> std::result::Result<
|
|
tonic::Response<openshell_core::proto::HttpRequestResult>,
|
|
tonic::Status,
|
|
> {
|
|
SupervisorMiddleware::evaluate_http_request(self, request).await
|
|
}
|
|
|
|
async fn open_websocket_session(
|
|
&self,
|
|
receiver: tokio::sync::mpsc::Receiver<openshell_core::proto::WebSocketSessionEvent>,
|
|
) -> std::result::Result<WebSocketResponseStream, tonic::Status> {
|
|
Ok(
|
|
self.websocket_stream(
|
|
tokio_stream::wrappers::ReceiverStream::new(receiver).map(Ok),
|
|
),
|
|
)
|
|
}
|
|
}
|
|
|
|
fn websocket_entry(
|
|
name: &str,
|
|
implementation: &str,
|
|
order: i32,
|
|
on_error: OnError,
|
|
) -> ChainEntry {
|
|
ChainEntry {
|
|
name: name.into(),
|
|
implementation: implementation.into(),
|
|
order,
|
|
config: prost_types::Struct::default(),
|
|
on_error,
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn empty_websocket_chain_skips_describe_and_preflight() {
|
|
let service = OpenAiRedactionService::default();
|
|
let describe_calls = Arc::clone(&service.describe_calls);
|
|
let observed_preflight = Arc::clone(&service.preflight);
|
|
let runner = ChainRunner::from_endpoint(Arc::new(service));
|
|
|
|
let outcome = runner
|
|
.preflight_websocket(&[], websocket_preflight_input("empty-chain"))
|
|
.await
|
|
.expect("empty preflight");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.terminal_reason, None);
|
|
assert!(outcome.session.is_none());
|
|
assert!(outcome.invocations.is_empty());
|
|
assert_eq!(describe_calls.load(std::sync::atomic::Ordering::SeqCst), 0);
|
|
assert!(observed_preflight.lock().expect("preflight lock").is_none());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn http_only_attachments_are_reported_but_do_not_select_websocket_stages() {
|
|
for on_error in [OnError::FailClosed, OnError::FailOpen] {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(ScriptedService {
|
|
manifest_name: "test/http-only".into(),
|
|
max_body_bytes: 4096,
|
|
result: allow_result(),
|
|
}));
|
|
let chain = [ChainEntry {
|
|
name: "http-guard".into(),
|
|
implementation: "test/http-only".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error,
|
|
}];
|
|
|
|
let outcome = runner
|
|
.preflight_websocket(
|
|
&chain,
|
|
websocket_preflight_input(format!("http-only-{on_error:?}")),
|
|
)
|
|
.await
|
|
.expect("HTTP-only attachment coverage");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.terminal_reason, None);
|
|
assert!(outcome.session.is_none());
|
|
assert!(outcome.invocations.is_empty());
|
|
assert_eq!(
|
|
outcome.coverage,
|
|
[WebSocketCoverage {
|
|
config_name: "http-guard".into(),
|
|
implementation: "test/http-only".into(),
|
|
state: WebSocketCoverageState::BindingNotSelected,
|
|
sequence: None,
|
|
message_type: None,
|
|
original_size: 0,
|
|
}]
|
|
);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn http_only_attachment_allows_33_requested_subprotocols() {
|
|
let runner = ChainRunner::new_protobuf_for_tests(Arc::new(ScriptedService {
|
|
manifest_name: "test/http-only".into(),
|
|
max_body_bytes: 4096,
|
|
result: allow_result(),
|
|
}));
|
|
let chain = [ChainEntry {
|
|
name: "http-guard".into(),
|
|
implementation: "test/http-only".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
}];
|
|
let mut input = websocket_preflight_input("http-only-many-subprotocols");
|
|
input.requested_subprotocols = (0..33)
|
|
.map(|index| format!("subprotocol-{index}"))
|
|
.collect();
|
|
|
|
let outcome = runner
|
|
.preflight_websocket(&chain, input)
|
|
.await
|
|
.expect("HTTP-only attachment must not validate a WebSocket middleware envelope");
|
|
|
|
assert!(outcome.allowed);
|
|
assert_eq!(outcome.terminal_reason, None);
|
|
assert!(outcome.session.is_none());
|
|
assert!(outcome.invocations.is_empty());
|
|
assert_eq!(
|
|
outcome.coverage,
|
|
[WebSocketCoverage {
|
|
config_name: "http-guard".into(),
|
|
implementation: "test/http-only".into(),
|
|
state: WebSocketCoverageState::BindingNotSelected,
|
|
sequence: None,
|
|
message_type: None,
|
|
original_size: 0,
|
|
}]
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn unsupported_binary_messages_advance_sequence_without_applying_on_error() {
|
|
for on_error in [OnError::FailClosed, OnError::FailOpen] {
|
|
let runner = builtin_runner();
|
|
let chain = [entry("regex-redactor", on_error)];
|
|
let preflight = runner
|
|
.preflight_websocket(
|
|
&chain,
|
|
websocket_preflight_input(format!("binary-{on_error:?}")),
|
|
)
|
|
.await
|
|
.expect("preflight");
|
|
let mut session = preflight.session.expect("built-in inspects text");
|
|
assert!(session.start("").await.allowed);
|
|
|
|
let coverage = session.observe_unsupported_message(WebSocketMessageType::Binary, 23);
|
|
assert_eq!(
|
|
coverage,
|
|
[WebSocketCoverage {
|
|
config_name: "regex-redactor".into(),
|
|
implementation: BUILTIN_REGEX.into(),
|
|
state: WebSocketCoverageState::UnsupportedMessageType,
|
|
sequence: Some(1),
|
|
message_type: Some(WebSocketMessageType::Binary),
|
|
original_size: 23,
|
|
}]
|
|
);
|
|
|
|
let text = session.evaluate_text(r#"{"input":"safe"}"#.into()).await;
|
|
assert!(text.allowed);
|
|
assert_eq!(text.invocations[0].sequence, Some(2));
|
|
assert!(!text.invocations[0].failed);
|
|
|
|
session
|
|
.end(openshell_core::proto::MiddlewareSessionEndReason::Normal)
|
|
.await;
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn explicit_websocket_preflight_denial_is_authoritative_for_both_error_modes() {
|
|
use openshell_core::proto::MiddlewareSessionEndReason;
|
|
|
|
for on_error in [OnError::FailOpen, OnError::FailClosed] {
|
|
let (session_ends_tx, mut session_ends_rx) = tokio::sync::mpsc::unbounded_channel();
|
|
let runner = ChainRunner::from_endpoint(Arc::new(OpenAiRedactionService {
|
|
deny: true,
|
|
preflight_reason: "contains sensitive request data".into(),
|
|
preflight_reason_code: "upgrade_blocked".into(),
|
|
preflight_findings: vec![Finding {
|
|
r#type: "content.sensitive".into(),
|
|
label: "Sensitive content".into(),
|
|
count: 1,
|
|
confidence: "high".into(),
|
|
severity: "high".into(),
|
|
}],
|
|
preflight_metadata: HashMap::from([("policy_version".into(), "1".into())]),
|
|
session_ends: Some(session_ends_tx),
|
|
..Default::default()
|
|
}));
|
|
|
|
let outcome = runner
|
|
.preflight_websocket(
|
|
&[websocket_entry(
|
|
"deny-upgrade",
|
|
"test/openai-websocket-redactor",
|
|
0,
|
|
on_error,
|
|
)],
|
|
websocket_preflight_input("explicit-denial"),
|
|
)
|
|
.await
|
|
.expect("denied preflight");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(
|
|
outcome.terminal_reason,
|
|
Some(MiddlewareSessionEndReason::MiddlewareDenial)
|
|
);
|
|
assert_eq!(
|
|
outcome.reason,
|
|
"middleware_denied:deny-upgrade:upgrade_blocked"
|
|
);
|
|
assert_eq!(
|
|
outcome
|
|
.denial
|
|
.as_ref()
|
|
.map(|denial| denial.config_name.as_str()),
|
|
Some("deny-upgrade")
|
|
);
|
|
assert_eq!(
|
|
outcome
|
|
.denial
|
|
.as_ref()
|
|
.and_then(|denial| denial.reason_code.as_deref()),
|
|
Some("upgrade_blocked")
|
|
);
|
|
assert_eq!(
|
|
outcome.invocations[0].reason_code.as_deref(),
|
|
Some("upgrade_blocked")
|
|
);
|
|
assert_eq!(outcome.findings.len(), 1);
|
|
assert_eq!(outcome.findings[0].middleware, "deny-upgrade");
|
|
assert_eq!(outcome.findings[0].finding.r#type, "content.sensitive");
|
|
assert_eq!(outcome.metadata["deny-upgrade"]["policy_version"], "1");
|
|
assert!(!outcome.reason.contains("sensitive request data"));
|
|
assert!(outcome.session.is_none());
|
|
assert_eq!(outcome.invocations.len(), 1);
|
|
assert_eq!(
|
|
outcome.invocations[0].outcome,
|
|
WebSocketInvocationOutcome::Deny
|
|
);
|
|
assert!(!outcome.invocations[0].failed);
|
|
assert_eq!(
|
|
session_ends_rx.recv().await,
|
|
Some(MiddlewareSessionEndReason::MiddlewareDenial)
|
|
);
|
|
assert!(
|
|
session_ends_rx.try_recv().is_err(),
|
|
"each opened stream receives at most one session_end"
|
|
);
|
|
}
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn mixed_websocket_preflight_denial_ends_every_opened_stage() {
|
|
use openshell_core::proto::MiddlewareSessionEndReason;
|
|
|
|
let (first_end_tx, mut first_end_rx) = tokio::sync::mpsc::unbounded_channel();
|
|
let (denier_end_tx, mut denier_end_rx) = tokio::sync::mpsc::unbounded_channel();
|
|
let (last_end_tx, mut last_end_rx) = tokio::sync::mpsc::unbounded_channel();
|
|
let endpoints: Vec<Arc<dyn InProcessMiddleware>> = vec![
|
|
in_process_endpoint(Arc::new(OpenAiRedactionService {
|
|
manifest_name: "test/first-inspector".into(),
|
|
session_ends: Some(first_end_tx),
|
|
..Default::default()
|
|
})),
|
|
in_process_endpoint(Arc::new(OpenAiRedactionService {
|
|
manifest_name: "test/denier".into(),
|
|
deny: true,
|
|
session_ends: Some(denier_end_tx),
|
|
..Default::default()
|
|
})),
|
|
in_process_endpoint(Arc::new(OpenAiRedactionService {
|
|
manifest_name: "test/last-inspector".into(),
|
|
session_ends: Some(last_end_tx),
|
|
..Default::default()
|
|
})),
|
|
];
|
|
let runner = ChainRunner::from_registry(
|
|
MiddlewareRegistry::connect_services(endpoints, Vec::new())
|
|
.await
|
|
.expect("connect mixed middleware services"),
|
|
);
|
|
let chain = [
|
|
websocket_entry("first", "test/first-inspector", 0, OnError::FailClosed),
|
|
websocket_entry("deny", "test/denier", 1, OnError::FailOpen),
|
|
websocket_entry("last", "test/last-inspector", 2, OnError::FailClosed),
|
|
];
|
|
|
|
let outcome = runner
|
|
.preflight_websocket(&chain, websocket_preflight_input("mixed-denial"))
|
|
.await
|
|
.expect("mixed preflight");
|
|
|
|
assert!(!outcome.allowed);
|
|
assert_eq!(
|
|
outcome.terminal_reason,
|
|
Some(MiddlewareSessionEndReason::MiddlewareDenial)
|
|
);
|
|
assert_eq!(
|
|
outcome
|
|
.invocations
|
|
.iter()
|
|
.map(|invocation| invocation.outcome)
|
|
.collect::<Vec<_>>(),
|
|
vec![
|
|
WebSocketInvocationOutcome::Inspect,
|
|
WebSocketInvocationOutcome::Deny,
|
|
WebSocketInvocationOutcome::Inspect,
|
|
]
|
|
);
|
|
for receiver in [&mut first_end_rx, &mut denier_end_rx, &mut last_end_rx] {
|
|
assert_eq!(
|
|
receiver.recv().await,
|
|
Some(MiddlewareSessionEndReason::MiddlewareDenial)
|
|
);
|
|
assert!(
|
|
receiver.try_recv().is_err(),
|
|
"each opened stream receives at most one session_end"
|
|
);
|
|
}
|
|
assert_eq!(
|
|
runner.registry.session_admission.available_permits(),
|
|
MAX_CONCURRENT_MIDDLEWARE_SESSIONS
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn builtin_regex_redacts_ordered_websocket_text_messages() {
|
|
let runner = builtin_runner();
|
|
let chain = [entry("regex-redactor", OnError::FailClosed)];
|
|
let preflight = runner
|
|
.preflight_websocket(
|
|
&chain,
|
|
WebSocketPreflightInput {
|
|
session_id: "builtin-regex-session".into(),
|
|
request_id: "request".into(),
|
|
sandbox_id: "sandbox".into(),
|
|
sandbox_name: "sandbox-name".into(),
|
|
workspace: "wrks-default".into(),
|
|
scheme: "wss".into(),
|
|
host: "api.openai.com".into(),
|
|
port: 443,
|
|
path: "/v1/responses".into(),
|
|
requested_subprotocols: vec!["realtime".into()],
|
|
},
|
|
)
|
|
.await
|
|
.expect("preflight");
|
|
assert!(preflight.allowed);
|
|
assert_eq!(
|
|
preflight.invocations[0].outcome,
|
|
WebSocketInvocationOutcome::Inspect
|
|
);
|
|
let mut session = preflight.session.expect("built-in chose to inspect");
|
|
assert!(session.start("realtime").await.allowed);
|
|
|
|
let original = r#"{"type":"response.create","response":{"input":"sk-ABCDEFGHIJKLMNOP"}}"#;
|
|
let redacted = session.evaluate_text(original.into()).await;
|
|
assert!(redacted.allowed);
|
|
assert_eq!(
|
|
redacted.payload,
|
|
r#"{"type":"response.create","response":{"input":"[REDACTED]"}}"#
|
|
);
|
|
assert_eq!(redacted.invocations[0].sequence, Some(1));
|
|
assert!(redacted.invocations[0].transformed);
|
|
assert_eq!(redacted.findings.len(), 1);
|
|
assert_eq!(redacted.findings[0].middleware, "regex-redactor");
|
|
assert_eq!(redacted.findings[0].finding.r#type, "regex.openai");
|
|
assert_eq!(
|
|
redacted.metadata["regex-redactor"]["regex_matches_replaced"],
|
|
"1"
|
|
);
|
|
|
|
let unchanged = session
|
|
.evaluate_text(r#"{"type":"response.cancel"}"#.into())
|
|
.await;
|
|
assert!(unchanged.allowed);
|
|
assert_eq!(unchanged.payload, r#"{"type":"response.cancel"}"#);
|
|
assert_eq!(unchanged.invocations[0].sequence, Some(2));
|
|
assert!(!unchanged.invocations[0].transformed);
|
|
assert!(unchanged.findings.is_empty());
|
|
assert!(unchanged.metadata.is_empty());
|
|
|
|
let oversized = session.evaluate_text("a".repeat(256 * 1024 + 1)).await;
|
|
assert!(!oversized.allowed);
|
|
assert_eq!(
|
|
oversized.reason,
|
|
"middleware_failed: request_message_over_capacity"
|
|
);
|
|
session
|
|
.end(openshell_core::proto::MiddlewareSessionEndReason::Normal)
|
|
.await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn builtin_regex_redacts_after_fail_open_message_capacity_gap() {
|
|
let runner = builtin_runner();
|
|
let chain = [entry("regex-redactor", OnError::FailOpen)];
|
|
let preflight = runner
|
|
.preflight_websocket(
|
|
&chain,
|
|
WebSocketPreflightInput {
|
|
session_id: "builtin-regex-gap-session".into(),
|
|
request_id: "request".into(),
|
|
sandbox_id: "sandbox".into(),
|
|
sandbox_name: "sandbox-name".into(),
|
|
workspace: "wrks-default".into(),
|
|
scheme: "wss".into(),
|
|
host: "api.openai.com".into(),
|
|
port: 443,
|
|
path: "/v1/responses".into(),
|
|
requested_subprotocols: vec!["realtime".into()],
|
|
},
|
|
)
|
|
.await
|
|
.expect("preflight");
|
|
assert!(preflight.allowed);
|
|
let mut session = preflight.session.expect("built-in chose to inspect");
|
|
assert!(session.start("realtime").await.allowed);
|
|
|
|
let oversized_payload = "a".repeat(256 * 1024 + 1);
|
|
let oversized = session.evaluate_text(oversized_payload.clone()).await;
|
|
assert!(oversized.allowed);
|
|
assert_eq!(oversized.payload, oversized_payload);
|
|
assert_eq!(oversized.invocations[0].sequence, Some(1));
|
|
assert_eq!(
|
|
oversized.invocations[0].outcome,
|
|
WebSocketInvocationOutcome::FailOpen
|
|
);
|
|
assert!(!oversized.invocations[0].stage_disabled);
|
|
|
|
let original = r#"{"type":"response.create","response":{"input":"sk-ABCDEFGHIJKLMNOP"}}"#;
|
|
let redacted = session.evaluate_text(original.into()).await;
|
|
assert!(redacted.allowed);
|
|
assert_eq!(
|
|
redacted.payload,
|
|
r#"{"type":"response.create","response":{"input":"[REDACTED]"}}"#
|
|
);
|
|
assert_eq!(redacted.invocations[0].sequence, Some(2));
|
|
assert_eq!(
|
|
redacted.invocations[0].outcome,
|
|
WebSocketInvocationOutcome::Allow
|
|
);
|
|
assert!(redacted.invocations[0].transformed);
|
|
assert!(!redacted.invocations[0].stage_disabled);
|
|
|
|
session
|
|
.end(openshell_core::proto::MiddlewareSessionEndReason::Normal)
|
|
.await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn in_process_websocket_endpoint_redacts_openai_event() {
|
|
let service = OpenAiRedactionService::default();
|
|
let runner = ChainRunner::from_endpoint(Arc::new(service));
|
|
let chain = [ChainEntry {
|
|
name: "openai-redactor".into(),
|
|
implementation: "test/openai-websocket-redactor".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
}];
|
|
let preflight = runner
|
|
.preflight_websocket(
|
|
&chain,
|
|
WebSocketPreflightInput {
|
|
session_id: "in-process-session".into(),
|
|
request_id: "request".into(),
|
|
sandbox_id: "sandbox".into(),
|
|
sandbox_name: "sandbox-name".into(),
|
|
workspace: "wrks-default".into(),
|
|
scheme: "wss".into(),
|
|
host: "api.openai.com".into(),
|
|
port: 443,
|
|
path: "/v1/responses".into(),
|
|
requested_subprotocols: vec!["realtime".into()],
|
|
},
|
|
)
|
|
.await
|
|
.expect("preflight");
|
|
assert!(preflight.allowed);
|
|
let mut session = preflight.session.expect("middleware chose to inspect");
|
|
assert!(session.start("realtime").await.allowed);
|
|
|
|
let original = r#"{"type":"response.create","response":{"input":"customer-secret"}}"#;
|
|
let outcome = session.evaluate_text(original.into()).await;
|
|
assert!(outcome.allowed);
|
|
assert_eq!(
|
|
outcome.payload,
|
|
r#"{"type":"response.create","response":{"input":"[REDACTED]"}}"#
|
|
);
|
|
assert!(outcome.invocations[0].transformed);
|
|
session
|
|
.end(openshell_core::proto::MiddlewareSessionEndReason::Normal)
|
|
.await;
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn openai_websocket_event_is_introspected_and_redacted() {
|
|
let service = OpenAiRedactionService::default();
|
|
let observed_preflight = Arc::clone(&service.preflight);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
.await
|
|
.expect("bind WebSocket middleware");
|
|
let address = listener.local_addr().expect("middleware address");
|
|
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
|
|
let server = tonic::transport::Server::builder()
|
|
.add_service(SupervisorMiddlewareServer::new(service))
|
|
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
|
|
let _ = shutdown_rx.await;
|
|
});
|
|
let server_task = tokio::spawn(server);
|
|
|
|
let mut registration = external_registration(1024);
|
|
registration.grpc_endpoint = format!("http://{address}");
|
|
let registry = MiddlewareRegistry::connect_services(Vec::new(), vec![registration])
|
|
.await
|
|
.expect("connect WebSocket middleware");
|
|
let runner = ChainRunner::from_registry(registry);
|
|
let chain = [ChainEntry {
|
|
name: "openai-redactor".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
}];
|
|
let described = runner
|
|
.describe_websocket_chain(&chain)
|
|
.await
|
|
.expect("describe WebSocket chain");
|
|
assert_eq!(
|
|
described[0].max_payload_bytes(),
|
|
1024,
|
|
"operator max_payload_bytes must cap WebSocket messages"
|
|
);
|
|
let preflight = runner
|
|
.preflight_websocket(
|
|
&chain,
|
|
WebSocketPreflightInput {
|
|
session_id: "ws-session".into(),
|
|
request_id: "request".into(),
|
|
sandbox_id: "sandbox".into(),
|
|
sandbox_name: "sandbox-name".into(),
|
|
workspace: "wrks-default".into(),
|
|
scheme: "wss".into(),
|
|
host: "api.openai.com".into(),
|
|
port: 443,
|
|
path: "/v1/responses".into(),
|
|
requested_subprotocols: vec!["realtime".into()],
|
|
},
|
|
)
|
|
.await
|
|
.expect("preflight");
|
|
assert!(preflight.allowed);
|
|
let mut session = preflight.session.expect("middleware chose to inspect");
|
|
assert!(session.start("realtime").await.allowed);
|
|
|
|
let original = r#"{"type":"response.create","response":{"input":"customer-secret"}}"#;
|
|
let outcome = session.evaluate_text(original.into()).await;
|
|
assert!(outcome.allowed);
|
|
let transformed = outcome.payload;
|
|
assert!(transformed.contains("[REDACTED]"));
|
|
assert!(!transformed.contains("customer-secret"));
|
|
assert!(outcome.invocations[0].transformed);
|
|
|
|
let observed = observed_preflight
|
|
.lock()
|
|
.expect("preflight lock")
|
|
.clone()
|
|
.expect("preflight observed");
|
|
let target = observed.target.expect("preflight target");
|
|
assert_eq!(target.scheme, "wss");
|
|
assert_eq!(target.host, "api.openai.com");
|
|
assert_eq!(target.port, 443);
|
|
assert_eq!(target.method, "GET");
|
|
assert_eq!(target.path, "/v1/responses");
|
|
assert!(target.query.is_empty());
|
|
assert_eq!(observed.requested_subprotocols, ["realtime"]);
|
|
session
|
|
.end(openshell_core::proto::MiddlewareSessionEndReason::Normal)
|
|
.await;
|
|
let _ = shutdown_tx.send(());
|
|
server_task
|
|
.await
|
|
.expect("join test middleware")
|
|
.expect("serve middleware");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn fail_open_disables_broken_websocket_stage_for_later_messages() {
|
|
let service = OpenAiRedactionService {
|
|
close_on_first_message: true,
|
|
..Default::default()
|
|
};
|
|
let observed_preflight = Arc::clone(&service.preflight);
|
|
let message_count = Arc::clone(&service.messages);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
.await
|
|
.expect("bind WebSocket middleware");
|
|
let address = listener.local_addr().expect("middleware address");
|
|
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
|
|
let server = tonic::transport::Server::builder()
|
|
.add_service(SupervisorMiddlewareServer::new(service))
|
|
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
|
|
let _ = shutdown_rx.await;
|
|
});
|
|
let server_task = tokio::spawn(server);
|
|
|
|
let mut registration = external_registration(1024);
|
|
registration.grpc_endpoint = format!("http://{address}");
|
|
let registry = MiddlewareRegistry::connect_services(Vec::new(), vec![registration])
|
|
.await
|
|
.expect("connect WebSocket middleware");
|
|
let runner = ChainRunner::from_registry(registry);
|
|
let chain = [ChainEntry {
|
|
name: "openai-redactor".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailOpen,
|
|
}];
|
|
let preflight = runner
|
|
.preflight_websocket(
|
|
&chain,
|
|
WebSocketPreflightInput {
|
|
session_id: "ws-session".into(),
|
|
request_id: "request".into(),
|
|
sandbox_id: "sandbox".into(),
|
|
sandbox_name: "sandbox-name".into(),
|
|
workspace: "wrks-default".into(),
|
|
scheme: "ws".into(),
|
|
host: "api.openai.com".into(),
|
|
port: 80,
|
|
path: "/v1/responses".into(),
|
|
requested_subprotocols: Vec::new(),
|
|
},
|
|
)
|
|
.await
|
|
.expect("preflight");
|
|
let mut session = preflight.session.expect("middleware chose to inspect");
|
|
assert!(session.start("").await.allowed);
|
|
assert_eq!(
|
|
observed_preflight
|
|
.lock()
|
|
.expect("preflight lock")
|
|
.as_ref()
|
|
.expect("preflight observed")
|
|
.target
|
|
.as_ref()
|
|
.expect("preflight target")
|
|
.scheme,
|
|
"ws"
|
|
);
|
|
|
|
let first = session
|
|
.evaluate_text(r#"{"type":"response.create"}"#.into())
|
|
.await;
|
|
assert!(first.allowed, "fail-open should bypass the broken stage");
|
|
assert_eq!(first.invocations.len(), 1);
|
|
assert!(first.invocations[0].failed);
|
|
assert!(first.invocations[0].stage_disabled);
|
|
assert_eq!(
|
|
runner.registry.session_admission.available_permits(),
|
|
MAX_CONCURRENT_MIDDLEWARE_SESSIONS,
|
|
"the final disabled stage must release persistent session capacity"
|
|
);
|
|
|
|
let mut work = Vec::new();
|
|
for _ in 0..MAX_CONCURRENT_MIDDLEWARE_WORK {
|
|
work.push(
|
|
runner
|
|
.reserve_middleware_work_admission()
|
|
.await
|
|
.expect("fill middleware work budget"),
|
|
);
|
|
}
|
|
|
|
let second = session
|
|
.evaluate_text(r#"{"type":"response.cancel"}"#.into())
|
|
.now_or_never()
|
|
.expect("fully disabled session must bypass without waiting for work admission");
|
|
assert!(second.allowed);
|
|
assert!(
|
|
second.invocations.is_empty(),
|
|
"disabled stage must not be called again in this session"
|
|
);
|
|
assert_eq!(message_count.load(std::sync::atomic::Ordering::SeqCst), 1);
|
|
drop(work);
|
|
|
|
session
|
|
.end(openshell_core::proto::MiddlewareSessionEndReason::Normal)
|
|
.await;
|
|
let _ = shutdown_tx.send(());
|
|
server_task
|
|
.await
|
|
.expect("join test middleware")
|
|
.expect("serve middleware");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn mixed_websocket_session_keeps_message_admission_while_one_stage_is_active() {
|
|
let broken = Arc::new(OpenAiRedactionService {
|
|
close_on_first_message: true,
|
|
..Default::default()
|
|
});
|
|
let mut endpoints = services();
|
|
endpoints.push(in_process_endpoint(broken));
|
|
let runner = ChainRunner::from_registry(
|
|
MiddlewareRegistry::connect_services(endpoints, Vec::new())
|
|
.await
|
|
.expect("connect mixed middleware services"),
|
|
);
|
|
let broken_entry = ChainEntry {
|
|
name: "best-effort-remote".into(),
|
|
implementation: "test/openai-websocket-redactor".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailOpen,
|
|
};
|
|
let mut regex_entry = entry("required-regex", OnError::FailClosed);
|
|
regex_entry.order = 1;
|
|
let preflight = runner
|
|
.preflight_websocket(
|
|
&[broken_entry, regex_entry],
|
|
websocket_preflight_input("mixed-active"),
|
|
)
|
|
.await
|
|
.expect("mixed preflight");
|
|
let mut session = preflight.session.expect("both stages inspect");
|
|
assert!(session.start("").await.allowed);
|
|
|
|
let first = session
|
|
.evaluate_text(r#"{"input":"sk-ABCDEFGHIJKLMNOP"}"#.into())
|
|
.await;
|
|
assert!(first.allowed);
|
|
assert!(first.invocations[0].stage_disabled);
|
|
assert_eq!(
|
|
first.invocations[1].outcome,
|
|
WebSocketInvocationOutcome::Allow
|
|
);
|
|
|
|
let mut work = Vec::new();
|
|
for _ in 0..MAX_CONCURRENT_MIDDLEWARE_WORK {
|
|
work.push(
|
|
runner
|
|
.reserve_middleware_work_admission()
|
|
.await
|
|
.expect("fill middleware work budget"),
|
|
);
|
|
}
|
|
assert!(
|
|
session.admit_message().now_or_never().is_none(),
|
|
"an active remaining stage must still wait for message work admission"
|
|
);
|
|
drop(work);
|
|
|
|
let second = session
|
|
.evaluate_text(r#"{"input":"sk-QRSTUVWXYZabcdef"}"#.into())
|
|
.await;
|
|
assert!(second.allowed);
|
|
assert_eq!(second.invocations.len(), 1);
|
|
assert_eq!(
|
|
second.invocations[0].config_name, "required-regex",
|
|
"disabled stage must stay bypassed while the active stage continues"
|
|
);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn websocket_admission_wait_queue_is_bounded() {
|
|
let runner = ChainRunner::default();
|
|
let mut active = Vec::new();
|
|
for _ in 0..MAX_CONCURRENT_MIDDLEWARE_WORK {
|
|
active.push(
|
|
runner
|
|
.reserve_middleware_work_admission()
|
|
.await
|
|
.expect("active admission"),
|
|
);
|
|
}
|
|
|
|
let mut waiters = Vec::new();
|
|
for _ in 0..MAX_QUEUED_MIDDLEWARE_WORK {
|
|
let runner = runner.clone();
|
|
waiters.push(tokio::spawn(async move {
|
|
runner.reserve_middleware_work().await
|
|
}));
|
|
}
|
|
while runner.registry.work_admission_waiters.available_permits() != 0 {
|
|
tokio::task::yield_now().await;
|
|
}
|
|
|
|
let overflow = runner
|
|
.reserve_middleware_work()
|
|
.await
|
|
.expect("admission outcome");
|
|
assert!(matches!(
|
|
overflow,
|
|
MiddlewareWorkAdmissionOutcome::QueueExhausted
|
|
));
|
|
runner
|
|
.reserve_middleware_work_admission()
|
|
.await
|
|
.expect_err("WebSocket callers retain their existing failure path");
|
|
|
|
drop(active);
|
|
for waiter in waiters {
|
|
let admission = waiter
|
|
.await
|
|
.expect("waiter task")
|
|
.expect("queued admission after capacity is released");
|
|
assert!(matches!(
|
|
admission,
|
|
MiddlewareWorkAdmissionOutcome::Admitted(_)
|
|
));
|
|
}
|
|
assert_eq!(
|
|
runner.registry.work_admission.available_permits(),
|
|
MAX_CONCURRENT_MIDDLEWARE_WORK
|
|
);
|
|
assert_eq!(
|
|
runner.registry.work_admission_waiters.available_permits(),
|
|
MAX_QUEUED_MIDDLEWARE_WORK
|
|
);
|
|
|
|
let mut recovered = Vec::new();
|
|
for _ in 0..MAX_CONCURRENT_MIDDLEWARE_WORK {
|
|
recovered.push(
|
|
runner
|
|
.reserve_middleware_work_admission()
|
|
.await
|
|
.expect("recovered active admission"),
|
|
);
|
|
}
|
|
assert_eq!(recovered.len(), MAX_CONCURRENT_MIDDLEWARE_WORK);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn websocket_queue_exhaustion_remains_a_protocol_failure() {
|
|
let runner = builtin_runner();
|
|
let chain = [entry("regex-redactor", OnError::FailClosed)];
|
|
let preflight = runner
|
|
.preflight_websocket(&chain, websocket_preflight_input("established"))
|
|
.await
|
|
.expect("initial preflight");
|
|
let mut session = preflight.session.expect("built-in inspects session");
|
|
assert!(session.start("").await.allowed);
|
|
|
|
let mut active = Vec::new();
|
|
for _ in 0..MAX_CONCURRENT_MIDDLEWARE_WORK {
|
|
active.push(
|
|
runner
|
|
.reserve_middleware_work_admission()
|
|
.await
|
|
.expect("fill active work"),
|
|
);
|
|
}
|
|
let mut waiters = Vec::new();
|
|
for _ in 0..MAX_QUEUED_MIDDLEWARE_WORK {
|
|
let runner = runner.clone();
|
|
waiters.push(tokio::spawn(async move {
|
|
runner.reserve_middleware_work().await
|
|
}));
|
|
}
|
|
while runner.registry.work_admission_waiters.available_permits() != 0 {
|
|
tokio::task::yield_now().await;
|
|
}
|
|
|
|
let preflight_overflow = runner
|
|
.preflight_websocket(&chain, websocket_preflight_input("preflight-overflow"))
|
|
.await;
|
|
assert!(
|
|
preflight_overflow.is_err(),
|
|
"preflight exhaustion remains an outer HTTP failure"
|
|
);
|
|
session
|
|
.admit_message()
|
|
.await
|
|
.expect_err("established message exhaustion remains a typed termination input");
|
|
|
|
for waiter in waiters {
|
|
waiter.abort();
|
|
}
|
|
drop(active);
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn websocket_preflight_skip_removes_stage_without_message_calls() {
|
|
let (session_ends_tx, mut session_ends_rx) = tokio::sync::mpsc::unbounded_channel();
|
|
let service = OpenAiRedactionService {
|
|
skip: true,
|
|
session_ends: Some(session_ends_tx),
|
|
..Default::default()
|
|
};
|
|
let message_count = Arc::clone(&service.messages);
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
.await
|
|
.expect("bind WebSocket middleware");
|
|
let address = listener.local_addr().expect("middleware address");
|
|
let (shutdown_tx, shutdown_rx) = tokio::sync::oneshot::channel();
|
|
let server = tonic::transport::Server::builder()
|
|
.add_service(SupervisorMiddlewareServer::new(service))
|
|
.serve_with_incoming_shutdown(TcpListenerStream::new(listener), async {
|
|
let _ = shutdown_rx.await;
|
|
});
|
|
let server_task = tokio::spawn(server);
|
|
let mut registration = external_registration(1024);
|
|
registration.grpc_endpoint = format!("http://{address}");
|
|
let runner = ChainRunner::from_registry(
|
|
MiddlewareRegistry::connect_services(Vec::new(), vec![registration])
|
|
.await
|
|
.expect("connect middleware"),
|
|
);
|
|
let result = runner
|
|
.preflight_websocket(
|
|
&[ChainEntry {
|
|
name: "scope".into(),
|
|
implementation: "local-guard-service".into(),
|
|
order: 0,
|
|
config: prost_types::Struct::default(),
|
|
on_error: OnError::FailClosed,
|
|
}],
|
|
WebSocketPreflightInput {
|
|
session_id: "session".into(),
|
|
request_id: "request".into(),
|
|
sandbox_id: "sandbox".into(),
|
|
sandbox_name: "sandbox-name".into(),
|
|
workspace: "wrks-default".into(),
|
|
scheme: "wss".into(),
|
|
host: "api.openai.com".into(),
|
|
port: 443,
|
|
path: "/v1/responses".into(),
|
|
requested_subprotocols: Vec::new(),
|
|
},
|
|
)
|
|
.await
|
|
.expect("preflight");
|
|
assert!(result.allowed);
|
|
assert!(result.session.is_none());
|
|
assert_eq!(
|
|
result.invocations[0].outcome,
|
|
WebSocketInvocationOutcome::Skip
|
|
);
|
|
assert_eq!(message_count.load(std::sync::atomic::Ordering::SeqCst), 0);
|
|
assert_eq!(
|
|
runner.registry.session_admission.available_permits(),
|
|
MAX_CONCURRENT_MIDDLEWARE_SESSIONS,
|
|
"all-skip preflight must not retain session capacity"
|
|
);
|
|
assert_eq!(
|
|
tokio::time::timeout(Duration::from_secs(1), session_ends_rx.recv())
|
|
.await
|
|
.expect("skipped stage must receive session_end"),
|
|
Some(openshell_core::proto::MiddlewareSessionEndReason::StageSkipped)
|
|
);
|
|
assert!(
|
|
session_ends_rx.try_recv().is_err(),
|
|
"a skipped stage receives at most one session_end"
|
|
);
|
|
let _ = shutdown_tx.send(());
|
|
server_task
|
|
.await
|
|
.expect("join middleware")
|
|
.expect("serve middleware");
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn websocket_session_budget_caps_idle_inspecting_sessions_and_releases_on_end() {
|
|
let runner = builtin_runner();
|
|
let chain = [entry("regex-redactor", OnError::FailClosed)];
|
|
let mut sessions = Vec::new();
|
|
for index in 0..MAX_CONCURRENT_MIDDLEWARE_SESSIONS {
|
|
let preflight = runner
|
|
.preflight_websocket(
|
|
&chain,
|
|
websocket_preflight_input(format!("session-{index}")),
|
|
)
|
|
.await
|
|
.expect("admit inspecting session");
|
|
assert!(preflight.allowed);
|
|
sessions.push(preflight.session.expect("built-in inspects session"));
|
|
}
|
|
assert_eq!(runner.registry.session_admission.available_permits(), 0);
|
|
|
|
let overflow = runner
|
|
.preflight_websocket(&chain, websocket_preflight_input("overflow"))
|
|
.await
|
|
.expect("capacity exhaustion is a typed preflight outcome");
|
|
assert!(!overflow.allowed);
|
|
assert!(overflow.session.is_none());
|
|
assert!(overflow.session_capacity_exhausted);
|
|
assert_eq!(
|
|
overflow.invocations[0].outcome,
|
|
WebSocketInvocationOutcome::FailClosed
|
|
);
|
|
|
|
sessions
|
|
.pop()
|
|
.expect("retained session")
|
|
.end(openshell_core::proto::MiddlewareSessionEndReason::Normal)
|
|
.await;
|
|
assert_eq!(runner.registry.session_admission.available_permits(), 1);
|
|
|
|
let replacement = runner
|
|
.preflight_websocket(&chain, websocket_preflight_input("replacement"))
|
|
.await
|
|
.expect("released session capacity is reusable");
|
|
assert!(replacement.allowed);
|
|
assert!(replacement.session.is_some());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn websocket_session_budget_survives_registry_replacement() {
|
|
let runner = builtin_runner();
|
|
let chain = [entry("regex-redactor", OnError::FailClosed)];
|
|
let mut sessions = Vec::new();
|
|
for index in 0..MAX_CONCURRENT_MIDDLEWARE_SESSIONS {
|
|
let preflight = runner
|
|
.preflight_websocket(
|
|
&chain,
|
|
websocket_preflight_input(format!("old-generation-{index}")),
|
|
)
|
|
.await
|
|
.expect("admit old-generation session");
|
|
sessions.push(preflight.session.expect("built-in inspects session"));
|
|
}
|
|
|
|
let replacement_registry = MiddlewareRegistry::connect_services(services(), Vec::new())
|
|
.await
|
|
.expect("connect replacement registry");
|
|
let replacement = runner.with_replacement_registry(replacement_registry);
|
|
let overflow = replacement
|
|
.preflight_websocket(&chain, websocket_preflight_input("new-generation-overflow"))
|
|
.await
|
|
.expect("capacity exhaustion is a typed preflight outcome");
|
|
assert!(!overflow.allowed);
|
|
assert!(overflow.session_capacity_exhausted);
|
|
|
|
sessions
|
|
.pop()
|
|
.expect("retained old-generation session")
|
|
.end(openshell_core::proto::MiddlewareSessionEndReason::PolicyReload)
|
|
.await;
|
|
let admitted = replacement
|
|
.preflight_websocket(&chain, websocket_preflight_input("new-generation-admitted"))
|
|
.await
|
|
.expect("released capacity is reusable after replacement");
|
|
assert!(admitted.allowed);
|
|
assert!(admitted.session.is_some());
|
|
}
|
|
|
|
#[tokio::test]
|
|
async fn websocket_session_capacity_exhaustion_honors_mixed_on_error() {
|
|
let runner = builtin_runner();
|
|
let mut held = Vec::new();
|
|
for _ in 0..MAX_CONCURRENT_MIDDLEWARE_SESSIONS {
|
|
match runner.try_reserve_middleware_session() {
|
|
MiddlewareSessionAdmission::Admitted(admission) => held.push(admission),
|
|
MiddlewareSessionAdmission::AtCapacity => {
|
|
panic!("session budget exhausted before platform limit")
|
|
}
|
|
}
|
|
}
|
|
|
|
let mut first = entry("best-effort-a", OnError::FailOpen);
|
|
first.order = 1;
|
|
let mut second = entry("best-effort-b", OnError::FailOpen);
|
|
second.order = 2;
|
|
let all_fail_open = runner
|
|
.preflight_websocket(
|
|
&[first.clone(), second.clone()],
|
|
websocket_preflight_input("all-fail-open"),
|
|
)
|
|
.await
|
|
.expect("all-fail-open capacity outcome");
|
|
assert!(all_fail_open.allowed);
|
|
assert!(all_fail_open.session.is_none());
|
|
assert!(all_fail_open.session_capacity_exhausted);
|
|
assert!(
|
|
all_fail_open
|
|
.invocations
|
|
.iter()
|
|
.all(|invocation| invocation.outcome == WebSocketInvocationOutcome::FailOpen)
|
|
);
|
|
|
|
second.on_error = OnError::FailClosed;
|
|
let mixed = runner
|
|
.preflight_websocket(&[first, second], websocket_preflight_input("mixed"))
|
|
.await
|
|
.expect("mixed capacity outcome");
|
|
assert!(!mixed.allowed);
|
|
assert!(mixed.session.is_none());
|
|
assert!(mixed.session_capacity_exhausted);
|
|
assert_eq!(
|
|
mixed
|
|
.invocations
|
|
.iter()
|
|
.map(|invocation| invocation.outcome)
|
|
.collect::<Vec<_>>(),
|
|
[
|
|
WebSocketInvocationOutcome::FailOpen,
|
|
WebSocketInvocationOutcome::FailClosed,
|
|
]
|
|
);
|
|
drop(held);
|
|
}
|
|
}
|