mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-23 06:30:28 +08:00
feat(kubernetes): add proxy-pod isolation topology
Signed-off-by: Drew Newberry <anewberry@nvidia.com>
This commit is contained in:
Drew Newberry
@@ -69,28 +69,12 @@ mise run helm:skaffold:dev
|
||||
mise run helm:skaffold:run
|
||||
```
|
||||
|
||||
**Supervisor sidecar topology** (build once and leave running):
|
||||
```bash
|
||||
mise run helm:skaffold:run:sidecar
|
||||
```
|
||||
|
||||
**Supervisor sidecar topology with TLS/mTLS enabled** (build once and leave running):
|
||||
```bash
|
||||
mise run helm:skaffold:run:sidecar-mtls
|
||||
```
|
||||
|
||||
Both commands build the `gateway` and `supervisor` images and deploy the OpenShell Helm
|
||||
chart. The sidecar profile renders an `openshell-network-init` init container for
|
||||
nftables setup and an `openshell-supervisor-network` runtime sidecar for proxying.
|
||||
Binary-aware policy mode runs that sidecar as UID 0 with `SYS_PTRACE` and
|
||||
`DAC_READ_SEARCH`; relaxed mode can run it as the configured proxy UID, which
|
||||
must be at least `1000` and distinct from the workload UID. The
|
||||
sidecar-mTLS profile reuses `ci/values-sidecar.yaml` and restores
|
||||
`server.disableTls=false` inline for Skaffold. The `pkiInitJob` hook (a pre-install
|
||||
Job that runs `openshell-gateway generate-certs`) generates mTLS secrets on first
|
||||
install. The default Skaffold values export gateway and Kubernetes-driver traces to
|
||||
the collector service installed by `helm:k3s:create`. Envoy Gateway opt-in; see the
|
||||
Optional Add-ons section below.
|
||||
Both commands build the gateway and trusted runtime image and deploy the
|
||||
OpenShell Helm chart. Kubernetes sandboxes always use the proxy-pod topology:
|
||||
a capability-free sandbox Pod behind an empty-egress NetworkPolicy and a
|
||||
separate capability-free supervisor Deployment. The default Skaffold values
|
||||
export gateway and Kubernetes-driver traces to the collector service installed
|
||||
by `helm:k3s:create`. Envoy Gateway is opt-in; see the Optional Add-ons section.
|
||||
|
||||
The gateway Service uses ClusterIP. Access is via Envoy Gateway (port `8080`) or
|
||||
the unified local forwarding task:
|
||||
@@ -102,8 +86,8 @@ mise run helm:k3s:forward
|
||||
The task forwards OTLP/gRPC to `http://127.0.0.1:4317` and the trace UI to
|
||||
`http://127.0.0.1:18888`. When Skaffold has deployed a Kubernetes gateway, it
|
||||
also forwards the gateway to `http://127.0.0.1:8090`; otherwise it continues
|
||||
with the collector ports only. A successful plaintext `helm:skaffold:run` or
|
||||
`helm:skaffold:run:sidecar` registers the gateway under the worktree-specific
|
||||
with the collector ports only. A successful plaintext `helm:skaffold:run`
|
||||
registers the gateway under the worktree-specific
|
||||
k3d cluster name and selects it as the active gateway. Keep the forwarding
|
||||
task running while using those endpoints.
|
||||
|
||||
@@ -134,8 +118,7 @@ create the Secret named `openshell-ha-pg` with a `uri` key, then run
|
||||
### TLS behaviour
|
||||
|
||||
`ci/values-skaffold.yaml` sets `server.disableTls: true`, so Skaffold-based deploys run
|
||||
plaintext by default. To test sidecar topology with TLS enabled, use
|
||||
`mise run helm:skaffold:run:sidecar-mtls`.
|
||||
plaintext by default. Remove that override to test gateway TLS/mTLS locally.
|
||||
|
||||
| Mode | `server.disableTls` | Gateway scheme |
|
||||
|------|---------------------|----------------|
|
||||
@@ -188,12 +171,6 @@ openshell sandbox list --gateway-endpoint https://localhost:8090
|
||||
mise run helm:skaffold:delete
|
||||
```
|
||||
|
||||
For a sidecar-profile deployment:
|
||||
|
||||
```bash
|
||||
mise run helm:skaffold:delete:sidecar
|
||||
```
|
||||
|
||||
### Delete the cluster entirely
|
||||
|
||||
```bash
|
||||
@@ -259,7 +236,7 @@ SPIFFE JWT-SVIDs for dynamic provider token grants:
|
||||
`openshell.local` and adds a `ClusterSPIFFEID` that maps sandbox pod
|
||||
annotations to `spiffe://openshell.local/openshell/sandbox/<sandbox-id>`.
|
||||
OpenShell mounts the SPIFFE CSI Workload API socket at
|
||||
`/spiffe-workload-api/spire-agent.sock` into sandbox pods for provider token
|
||||
`/spiffe-workload-api/spire-agent.sock` only into supervisor Pods for provider token
|
||||
grants. Supervisor-to-gateway authentication remains on the Kubernetes
|
||||
ServiceAccount bootstrap and gateway-minted sandbox JWT path; the selected
|
||||
Kubernetes compute driver validates the projected token before the gateway
|
||||
@@ -320,7 +297,6 @@ for dependencies still declared in `Chart.yaml`.
|
||||
| `deploy/helm/openshell/ci/values-gateway.yaml` | Envoy Gateway GRPCRoute + Gateway overlay |
|
||||
| `deploy/helm/openshell/ci/values-high-availability.yaml` | HA test overlay (`replicaCount: 2` with external PostgreSQL Secret) |
|
||||
| `deploy/helm/openshell/ci/values-keycloak.yaml` | Keycloak OIDC overlay |
|
||||
| `deploy/helm/openshell/ci/values-sidecar.yaml` | Supervisor sidecar topology overlay for Kubernetes e2e/dev |
|
||||
| `deploy/helm/openshell/ci/values-spire.yaml` | SPIFFE/SPIRE provider token grant overlay |
|
||||
| `deploy/helm/openshell/ci/values-spire-stack.yaml` | SPIRE hardened chart values for local dev |
|
||||
| `deploy/helm/openshell/ci/values-tls-disabled.yaml` | Lint-only: TLS + auth disabled (reverse-proxy edge termination) |
|
||||
|
||||
@@ -406,16 +406,12 @@ jobs:
|
||||
include:
|
||||
- agent_sandbox_api: v1beta1
|
||||
agent_sandbox_version: v0.5.0
|
||||
topology: combined
|
||||
extra_helm_values: ""
|
||||
topology: proxy-pod
|
||||
extra_helm_values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
- agent_sandbox_api: v1alpha1
|
||||
agent_sandbox_version: v0.4.6
|
||||
topology: combined
|
||||
extra_helm_values: ""
|
||||
- agent_sandbox_api: v1beta1
|
||||
agent_sandbox_version: v0.5.0
|
||||
topology: sidecar
|
||||
extra_helm_values: deploy/helm/openshell/ci/values-sidecar.yaml
|
||||
topology: proxy-pod
|
||||
extra_helm_values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
permissions:
|
||||
actions: read
|
||||
contents: read
|
||||
@@ -440,6 +436,7 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
job-name: Kubernetes E2E (workspace managed mode)
|
||||
e2e-task: e2e:kubernetes:workspace-managed
|
||||
extra-helm-values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-external-driver-e2e:
|
||||
@@ -457,6 +454,7 @@ jobs:
|
||||
gateway-artifact: openshell-gateway-plain-x86_64-unknown-linux-gnu
|
||||
external-driver-binary: openshell-driver-kubernetes
|
||||
cluster-images: supervisor
|
||||
extra-helm-values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-workspace-operator-e2e:
|
||||
@@ -471,6 +469,7 @@ jobs:
|
||||
image-tag: ${{ github.sha }}
|
||||
job-name: Kubernetes E2E (workspace operator mode)
|
||||
e2e-task: e2e:kubernetes:workspace-operator
|
||||
extra-helm-values: deploy/helm/openshell/ci/values-proxy-pod.yaml
|
||||
conformance-artifact-prefix: openshell-conformance
|
||||
|
||||
kubernetes-ha-e2e:
|
||||
|
||||
Generated
+3
@@ -4142,6 +4142,7 @@ dependencies = [
|
||||
"miette",
|
||||
"notify",
|
||||
"openshell-core",
|
||||
"openshell-isolation-interface",
|
||||
"openshell-otel",
|
||||
"openshell-otel-test-support",
|
||||
"openshell-policy",
|
||||
@@ -4149,6 +4150,8 @@ dependencies = [
|
||||
"opentelemetry_sdk",
|
||||
"prost",
|
||||
"prost-types",
|
||||
"rand 0.9.4",
|
||||
"rcgen",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"temp-env",
|
||||
|
||||
@@ -12,7 +12,6 @@ pub const OPT_ROOT: &str = "/opt/openshell";
|
||||
pub const ETC_ROOT: &str = "/etc/openshell";
|
||||
pub const TLS_ROOT: &str = "/etc/openshell-tls";
|
||||
pub const RUN_ROOT: &str = "/run/openshell";
|
||||
pub const SIDECAR_RUN_ROOT: &str = "/run/openshell-sidecar";
|
||||
pub const NETNS_MOUNT_ROOT: &str = "/run/netns";
|
||||
pub const NETNS_IPROUTE2_ROOT: &str = "/var/run/netns";
|
||||
|
||||
@@ -34,7 +33,6 @@ pub const CONTROL_ROOTS: &[&str] = &[
|
||||
ETC_ROOT,
|
||||
TLS_ROOT,
|
||||
RUN_ROOT,
|
||||
SIDECAR_RUN_ROOT,
|
||||
NETNS_MOUNT_ROOT,
|
||||
// The supervisor currently uses the conventional iproute2 spelling.
|
||||
NETNS_IPROUTE2_ROOT,
|
||||
@@ -52,9 +50,6 @@ pub const CONTAINER_POLICY_PATH: &str = "/etc/openshell/policy.yaml";
|
||||
pub const POLICY_ADVISOR_SKILL_PATH: &str = "/etc/openshell/skills/policy_advisor.md";
|
||||
|
||||
pub const SSH_SOCKET_PATH: &str = "/run/openshell/ssh.sock";
|
||||
pub const SIDECAR_CONTROL_SOCKET: &str = "/run/openshell-sidecar/control.sock";
|
||||
pub const SIDECAR_TLS_DIR: &str = "/etc/openshell-tls/proxy";
|
||||
pub const SIDECAR_CLIENT_TLS_DIR: &str = "/etc/openshell-tls/proxy/client";
|
||||
pub const CLIENT_TLS_DIR: &str = "/etc/openshell-tls/client";
|
||||
pub const SUPERVISOR_CA_CERT_PATH: &str = "/etc/openshell-tls/openshell-ca.pem";
|
||||
pub const SUPERVISOR_CA_BUNDLE_PATH: &str = "/etc/openshell-tls/ca-bundle.pem";
|
||||
@@ -92,9 +87,6 @@ mod tests {
|
||||
CONTAINER_POLICY_PATH,
|
||||
POLICY_ADVISOR_SKILL_PATH,
|
||||
SSH_SOCKET_PATH,
|
||||
SIDECAR_CONTROL_SOCKET,
|
||||
SIDECAR_TLS_DIR,
|
||||
SIDECAR_CLIENT_TLS_DIR,
|
||||
CLIENT_TLS_DIR,
|
||||
SUPERVISOR_CA_CERT_PATH,
|
||||
SUPERVISOR_CA_BUNDLE_PATH,
|
||||
|
||||
@@ -296,7 +296,6 @@ mod tests {
|
||||
"/etc/openshell-tls",
|
||||
"/run",
|
||||
"/run/openshell/cache",
|
||||
"/run/openshell-sidecar/control.sock",
|
||||
"/run/netns/project",
|
||||
"/var/run/netns/project",
|
||||
] {
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
//! Shared state controlling agent-driven policy proposals.
|
||||
//!
|
||||
//! Initialised once during sandbox start from the `agent_policy_proposals_enabled`
|
||||
//! setting and updated by the policy poll loop or authoritative sidecar control
|
||||
//! setting and updated by the policy poll loop or authoritative supervisor
|
||||
//! when the setting changes. Read by the `policy.local` route handler and by
|
||||
//! the skills installer to gate the agent-controlled mutation surface.
|
||||
|
||||
@@ -15,7 +15,7 @@ use std::sync::atomic::{AtomicBool, Ordering};
|
||||
///
|
||||
/// Clones point at the same atomic value, so the sandbox orchestrator can pass
|
||||
/// this into the process and network supervisors and then update it from the
|
||||
/// settings poll loop or sidecar control.
|
||||
/// settings poll loop or supervisor control.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AgentProposals {
|
||||
enabled: Arc<AtomicBool>,
|
||||
|
||||
@@ -171,8 +171,8 @@ impl ProviderCredentialState {
|
||||
/// Build a static provider state from an already-prepared child
|
||||
/// environment snapshot.
|
||||
///
|
||||
/// Kubernetes sidecar topology uses this in the process-only supervisor:
|
||||
/// the network sidecar owns provider credential resolvers and sends the
|
||||
/// The Kubernetes proxy-pod topology uses this in the sandbox process:
|
||||
/// the supervisor owns provider credential resolvers and sends the
|
||||
/// workload-facing env map over a local control channel. The process leaf
|
||||
/// must inject that map into child processes without re-placeholderizing it
|
||||
/// or holding the gateway-side resolver material.
|
||||
|
||||
@@ -131,10 +131,6 @@ impl MainProcessConfig {
|
||||
/// Deployment-controlled telemetry toggle propagated to the sandbox supervisor.
|
||||
pub const TELEMETRY_ENABLED: &str = "OPENSHELL_TELEMETRY_ENABLED";
|
||||
|
||||
/// Supervisor pod/runtime topology. Kubernetes sidecar mode sets this to
|
||||
/// `"sidecar"`; the default combined supervisor path omits it.
|
||||
pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY";
|
||||
|
||||
/// The isolation backend admitted by the deployment configuration (RFC 0012).
|
||||
///
|
||||
/// Delivered on a channel separate from the topology descriptor so descriptor
|
||||
@@ -142,9 +138,6 @@ pub const SUPERVISOR_TOPOLOGY: &str = "OPENSHELL_SUPERVISOR_TOPOLOGY";
|
||||
/// whenever a topology descriptor is supplied.
|
||||
pub const ADMITTED_ISOLATION_BACKEND: &str = "OPENSHELL_ADMITTED_ISOLATION_BACKEND";
|
||||
|
||||
/// Network enforcement backend selected by the compute driver.
|
||||
pub const NETWORK_ENFORCEMENT_MODE: &str = "OPENSHELL_NETWORK_ENFORCEMENT_MODE";
|
||||
|
||||
/// Comma-separated runtime networking capabilities supplied by the compute
|
||||
/// driver. Capabilities describe substrate the shared supervisor may activate;
|
||||
/// they never move policy evaluation into the driver.
|
||||
@@ -153,20 +146,6 @@ pub const NETWORK_RUNTIME_CAPABILITIES: &str = "OPENSHELL_NETWORK_RUNTIME_CAPABI
|
||||
/// Driver capability for policy-gated DNS and transparent TCP interception.
|
||||
pub const POLICY_DNS_TRANSPARENT_TCP_CAPABILITY: &str = "policy-dns-transparent-tcp";
|
||||
|
||||
/// Whether network policy evaluation must bind requests to the peer binary.
|
||||
///
|
||||
/// The default when unset is `"required"`. Kubernetes sidecar experiments may
|
||||
/// set this to `"relaxed"` to enforce endpoint and L7 policy without per-binary
|
||||
/// `/proc` identity binding.
|
||||
pub const NETWORK_BINARY_IDENTITY: &str = "OPENSHELL_NETWORK_BINARY_IDENTITY";
|
||||
|
||||
/// Unix socket used by Kubernetes sidecar topology for local coordination.
|
||||
///
|
||||
/// The network sidecar owns gateway credentials and serves policy/provider
|
||||
/// state over this socket instead of exposing gateway credentials to the agent
|
||||
/// container.
|
||||
pub const SIDECAR_CONTROL_SOCKET: &str = "OPENSHELL_SIDECAR_CONTROL_SOCKET";
|
||||
|
||||
/// Optional TLS server name override used when connecting to the gateway.
|
||||
pub const GATEWAY_TLS_SERVER_NAME: &str = "OPENSHELL_GATEWAY_TLS_SERVER_NAME";
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
openshell-core = { path = "../openshell-core", default-features = false }
|
||||
openshell-isolation-interface = { path = "../openshell-isolation-interface" }
|
||||
openshell-otel = { path = "../openshell-otel" }
|
||||
openshell-policy = { path = "../openshell-policy" }
|
||||
|
||||
@@ -37,6 +38,8 @@ tracing-subscriber = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
miette = { workspace = true }
|
||||
notify = "8"
|
||||
rand = { workspace = true }
|
||||
rcgen = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
openshell-otel-test-support = { path = "../openshell-otel-test-support" }
|
||||
|
||||
@@ -45,9 +45,43 @@ not a tenant isolation boundary.
|
||||
## Runtime Model
|
||||
|
||||
The gateway stores platform state and delegates sandbox workload creation to
|
||||
this driver. Kubernetes owns scheduling and pod lifecycle. The
|
||||
`openshell-sandbox` supervisor inside each workload owns agent isolation,
|
||||
credential injection, policy polling, logs, and the gateway relay.
|
||||
this driver. Kubernetes owns scheduling and pod lifecycle. The driver always
|
||||
uses the proxy-pod topology. The workload Pod runs `openshell-sandbox`, while a
|
||||
separate Deployment runs `openshell-supervisor`.
|
||||
|
||||
The sandbox owns the agent process, applies Landlock and child seccomp filters,
|
||||
identifies the binary behind each network syscall, and relays mediated streams
|
||||
to the supervisor. The supervisor authenticates to the gateway with a JWT,
|
||||
loads policy and provider state, performs destination and L7 authorization, and
|
||||
opens upstream connections. The workload receives no gateway credential,
|
||||
provider identity socket, or corporate-proxy credential.
|
||||
|
||||
Both Pods run as the namespace-resolved non-root UID/GID with
|
||||
`allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, and the runtime
|
||||
default seccomp profile. The sandbox installs a nested seccomp user-notification
|
||||
filter without requesting a capability in the Pod spec. Startup fails closed
|
||||
when the runtime blocks the required seccomp or Landlock operations.
|
||||
|
||||
The driver creates an empty-egress `NetworkPolicy` before it releases the
|
||||
workload Pod. A second rule permits only the paired supervisor Pod to reach the
|
||||
sandbox TLS port. The supervisor has normal egress for gateway, DNS, and
|
||||
policy-approved upstream connections. Set
|
||||
`proxy_pod.network_policy_enforced = true` only after verifying that the cluster
|
||||
CNI enforces ingress and egress `NetworkPolicy` for sandbox namespaces.
|
||||
|
||||
Each sandbox generation uses two immutable bootstrap Secrets. A trusted init
|
||||
container stages the sandbox bootstrap into memory, and the sandbox removes it
|
||||
before starting untrusted code. The other Secret is mounted only by the
|
||||
supervisor. The TLS channel binds the namespace, Sandbox CR, workload Pod,
|
||||
supervisor Deployment, and egress policy identities. Stop deletes the workload
|
||||
Pod and scales the supervisor to zero. Start rotates both Secrets and the
|
||||
supervisor Deployment before creating a new workload Pod. Delete retains the
|
||||
egress fence until workload deletion is confirmed.
|
||||
|
||||
Kubernetes policies are additive, and the API does not attest that the CNI
|
||||
enforces them. Keep sandbox namespaces administrative: untrusted principals
|
||||
must not create permissive policies, create Pods, read bootstrap Secrets, or
|
||||
spoof the pair labels.
|
||||
|
||||
## Sandbox Resource
|
||||
|
||||
@@ -95,73 +129,31 @@ mount attaches an existing PVC under `/sandbox`, which skips the default PVC.
|
||||
|
||||
## Credentials, TLS, and Relay
|
||||
|
||||
The driver injects gateway callback configuration, sandbox identity, TLS client
|
||||
material, and the supervisor SSH socket path into the workload. The callback
|
||||
endpoint is required because the sandbox namespace does not identify the
|
||||
Gateway Service; Helm renders it from the release topology, while standalone
|
||||
and raw TOML configurations must set it explicitly. Driver-owned values must
|
||||
override image-provided environment variables.
|
||||
Both Pods set `automountServiceAccountToken: false`. The supervisor receives an
|
||||
explicit audience-bound projected token for the one-shot `IssueSandboxToken`
|
||||
exchange. The driver verifies that token and the gateway returns the
|
||||
sandbox-scoped JWT used by the supervisor session. The sandbox Pod receives
|
||||
neither token.
|
||||
|
||||
Sandbox pods run as `service_account_name` and keep
|
||||
`automountServiceAccountToken: false`. The only Kubernetes token exposed to the
|
||||
supervisor is an explicit, audience-bound projected token mounted at
|
||||
`/var/run/secrets/openshell/token` for the one-shot `IssueSandboxToken`
|
||||
bootstrap exchange. The Kubernetes driver authenticates that token through the
|
||||
compute-driver protocol using its own `service_account_name` and workspace-mode
|
||||
namespace policy; the gateway receives only the verified sandbox ID.
|
||||
The gateway uses the supervisor relay for connect, exec, logs, and file sync.
|
||||
Sandbox Pods do not need direct external ingress for SSH.
|
||||
|
||||
The gateway uses the supervisor relay for connect, exec, and file sync. Sandbox
|
||||
pods do not need direct external ingress for SSH.
|
||||
|
||||
The driver forwards the canonical main-process specification to the process
|
||||
supervisor and sets pod `restartPolicy: Never`. Main-process environment
|
||||
overrides stay local to that child; the sidecar bootstrap retains the unmodified
|
||||
provider environment used by later exec, editor, and SFTP sessions.
|
||||
The driver sends the canonical main-process specification only to the
|
||||
supervisor. The supervisor passes admitted launch state over the protected
|
||||
channel. Provider environment updates apply to future exec sessions.
|
||||
|
||||
## Container Security Context
|
||||
|
||||
The default `combined` supervisor topology grants the sandbox agent container
|
||||
the Linux capabilities the supervisor needs for namespace setup and process,
|
||||
filesystem, and network policy enforcement.
|
||||
The sandbox, trusted bootstrap init container, and supervisor request no added
|
||||
Linux capability. They run as the same numeric non-root identity, disable
|
||||
privilege escalation, drop all capabilities, and inherit `RuntimeDefault`
|
||||
seccomp. The sandbox and agent must use the same complete UID, GID, and
|
||||
supplementary-group identity because the capability-free sandbox cannot change
|
||||
credentials after launch and must inspect its same-identity descendants.
|
||||
|
||||
The `sidecar` supervisor topology moves pod-level network setup into a root init
|
||||
container. In the default process/binary-aware mode, the long-lived network
|
||||
sidecar runs as UID 0 with `allowPrivilegeEscalation: false`, drops default
|
||||
Linux capabilities, and adds only `SYS_PTRACE` plus `DAC_READ_SEARCH` for
|
||||
cross-UID workload `/proc` inspection. The agent container also runs as the
|
||||
resolved sandbox UID/GID with `allowPrivilegeEscalation: false` and
|
||||
`capabilities.drop: ["ALL"]`.
|
||||
Set `sidecar.process_binary_aware_network_policy = false` to run the network
|
||||
sidecar as the configured non-root `sidecar.proxy_uid`, omit the extra `/proc`
|
||||
inspection capabilities, and enforce endpoint/L7 network policy without
|
||||
matching `policy.binaries`.
|
||||
In this mode OpenShell preserves gateway session and SSH behavior, but the
|
||||
process supervisor does not perform root-to-sandbox privilege dropping or
|
||||
supervisor identity mount isolation. It still applies Landlock filesystem policy
|
||||
and child seccomp filters where the kernel/runtime supports them. Network
|
||||
endpoint and L7 policy remain enforced by the network sidecar, and
|
||||
sidecar pods use a shared process namespace so the network sidecar can resolve
|
||||
process/binary identity through `/proc/<entrypoint-pid>`.
|
||||
|
||||
Sidecar mode keeps gateway credentials in the network sidecar. The agent
|
||||
container does not mount the projected service-account token used for sandbox
|
||||
token bootstrap, does not mount the sandbox client TLS secret, and does not get
|
||||
gateway callback environment variables. The process supervisor receives policy
|
||||
and provider environment state from the sidecar over a local control socket in
|
||||
the shared sidecar state volume. The sidecar accepts only the pre-workload
|
||||
process-supervisor connection, authenticates its UID/GID/PID with peer
|
||||
credentials, and removes the listener afterward. SSH relays use a Linux
|
||||
abstract socket whose peer PID must match that authenticated supervisor. Both
|
||||
supervisors exit if the control connection closes, coupling their container
|
||||
restart lifecycle before a new authoritative client can be established.
|
||||
|
||||
The driver uses the shared AppArmor model through `app_armor_profile`.
|
||||
Supported values are `Unconfined`, `RuntimeDefault`, and
|
||||
`Localhost/<profile-name>`; an empty or unset value omits
|
||||
`securityContext.appArmorProfile`. Docker and Podman translate the same values
|
||||
to OCI security options. Helm deployments default sandbox agent containers to
|
||||
`Unconfined` because runtime/default AppArmor profiles can block the
|
||||
supervisor's network namespace mount setup on AppArmor-enabled nodes.
|
||||
The workload Pod does not share host network, PID, IPC, or process namespaces.
|
||||
The driver uses a scheduling gate to inspect the admitted Pod and bind its UID
|
||||
into the bootstrap claims before kubelet starts it.
|
||||
|
||||
## GPU Support
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
pub use openshell_core::AppArmorProfile;
|
||||
pub use openshell_core::DynamicStringAllowlist as OperatorNamespaceAllowlist;
|
||||
use openshell_core::{ImagePullPolicy, config};
|
||||
use openshell_core::config;
|
||||
use serde::{Deserialize, Deserializer, Serialize};
|
||||
use std::collections::BTreeMap;
|
||||
#[cfg(test)]
|
||||
@@ -23,76 +22,38 @@ pub const DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME: &str = "default";
|
||||
/// Default storage size for the workspace PVC.
|
||||
pub const DEFAULT_WORKSPACE_STORAGE_SIZE: &str = "2Gi";
|
||||
|
||||
/// Default non-root UID for relaxed Kubernetes network supervisor sidecars.
|
||||
pub const DEFAULT_PROXY_UID: u32 = 1337;
|
||||
|
||||
/// How the supervisor binary is delivered into sandbox pods.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum SupervisorSideloadMethod {
|
||||
/// Mount the supervisor OCI image directly as a read-only volume
|
||||
/// (requires Kubernetes >= v1.33 with the `ImageVolume` feature gate,
|
||||
/// or >= v1.36 where it is GA).
|
||||
#[default]
|
||||
ImageVolume,
|
||||
/// Copy the binary via an init container and emptyDir volume.
|
||||
/// Works on all Kubernetes versions.
|
||||
InitContainer,
|
||||
/// Driver-owned requirements for the cross-pod RFC 0012 topology.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub struct KubernetesProxyPodConfig {
|
||||
/// Explicit operator assertion that the cluster CNI enforces
|
||||
/// `networking.k8s.io/v1` `NetworkPolicy` for the sandbox namespaces.
|
||||
pub network_policy_enforced: bool,
|
||||
/// TCP port exposed by the workload boundary to its paired control pod.
|
||||
pub boundary_port: u16,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SupervisorSideloadMethod {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::ImageVolume => f.write_str("image-volume"),
|
||||
Self::InitContainer => f.write_str("init-container"),
|
||||
impl Default for KubernetesProxyPodConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
network_policy_enforced: false,
|
||||
boundary_port: 5500,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for SupervisorSideloadMethod {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s {
|
||||
"image-volume" => Ok(Self::ImageVolume),
|
||||
"init-container" => Ok(Self::InitContainer),
|
||||
other => Err(format!(
|
||||
"unknown supervisor sideload method '{other}'; expected 'image-volume' or 'init-container'"
|
||||
)),
|
||||
impl KubernetesProxyPodConfig {
|
||||
pub fn validate(&self) -> Result<(), String> {
|
||||
if !self.network_policy_enforced {
|
||||
return Err(
|
||||
"proxy-pod topology requires proxy_pod.network_policy_enforced = true after the operator has verified CNI NetworkPolicy enforcement"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How the supervisor is arranged inside Kubernetes sandbox pods.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "kebab-case")]
|
||||
pub enum SupervisorTopology {
|
||||
/// Run networking and process supervision in the agent container.
|
||||
#[default]
|
||||
Combined,
|
||||
/// Run network supervision in a privileged sidecar and process supervision
|
||||
/// as a low-capability wrapper in the agent container.
|
||||
Sidecar,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for SupervisorTopology {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Combined => f.write_str("combined"),
|
||||
Self::Sidecar => f.write_str("sidecar"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl FromStr for SupervisorTopology {
|
||||
type Err = String;
|
||||
|
||||
fn from_str(s: &str) -> Result<Self, Self::Err> {
|
||||
match s {
|
||||
"combined" => Ok(Self::Combined),
|
||||
"sidecar" => Ok(Self::Sidecar),
|
||||
other => Err(format!("unknown topology '{other}'")),
|
||||
if self.boundary_port < 1024 {
|
||||
return Err("proxy_pod.boundary_port must be at least 1024".to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,62 +99,6 @@ impl FromStr for WorkspaceMode {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub struct KubernetesSidecarConfig {
|
||||
/// UID used by relaxed long-running network sidecars in `sidecar`
|
||||
/// topology. The network init container installs nftables rules that
|
||||
/// exempt this UID, so it must not match the sandbox workload UID.
|
||||
/// Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants
|
||||
/// the requested `/proc` inspection capabilities into the effective set.
|
||||
pub proxy_uid: u32,
|
||||
/// Require process/binary-aware network policy enforcement in sidecar
|
||||
/// topology. When disabled, the network sidecar runs as `proxy_uid`,
|
||||
/// drops the extra `/proc` inspection permissions, and evaluates
|
||||
/// endpoint/L7 policy without matching `policy.binaries`.
|
||||
pub process_binary_aware_network_policy: bool,
|
||||
}
|
||||
|
||||
impl Default for KubernetesSidecarConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
proxy_uid: DEFAULT_PROXY_UID,
|
||||
process_binary_aware_network_policy: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl KubernetesSidecarConfig {
|
||||
pub fn validate_proxy_uid(&self) -> Result<(), String> {
|
||||
if !(openshell_policy::MIN_SANDBOX_PROXY_UID..=openshell_policy::MAX_SANDBOX_UID)
|
||||
.contains(&self.proxy_uid)
|
||||
{
|
||||
return Err(format!(
|
||||
"sidecar.proxy_uid must be in range [{}, {}]",
|
||||
openshell_policy::MIN_SANDBOX_PROXY_UID,
|
||||
openshell_policy::MAX_SANDBOX_UID,
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn deserialize_optional_app_armor_profile<'de, D>(
|
||||
deserializer: D,
|
||||
) -> Result<Option<AppArmorProfile>, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let value = Option::<String>::deserialize(deserializer)?;
|
||||
match value.as_deref() {
|
||||
None | Some("") => Ok(None),
|
||||
Some(value) => value
|
||||
.parse::<AppArmorProfile>()
|
||||
.map(Some)
|
||||
.map_err(serde::de::Error::custom),
|
||||
}
|
||||
}
|
||||
|
||||
fn deserialize_provider_spiffe_workload_api_socket_path<'de, D>(
|
||||
deserializer: D,
|
||||
) -> Result<String, D::Error>
|
||||
@@ -227,33 +132,26 @@ pub struct KubernetesComputeConfig {
|
||||
/// operator mode. Hot-reloaded on change. Delivered via `ConfigMap` volume mount.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub operator_namespace_file: Option<String>,
|
||||
/// Kubernetes `ServiceAccount` assigned to sandbox pods and accepted by
|
||||
/// the driver's `TokenReview` bootstrap authenticator.
|
||||
/// Kubernetes `ServiceAccount` assigned to both topology Pods. Automatic
|
||||
/// token mounting is disabled; only the supervisor receives an explicit
|
||||
/// audience-bound projected token accepted by the bootstrap authenticator.
|
||||
pub service_account_name: String,
|
||||
pub default_image: String,
|
||||
/// Pull policy for sandbox images. Omit to use Kubernetes's image default.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub image_pull_policy: Option<ImagePullPolicy>,
|
||||
pub image_pull_policy: String,
|
||||
/// Kubernetes `imagePullSecrets` names attached to sandbox pods.
|
||||
pub image_pull_secrets: Vec<String>,
|
||||
/// Managed-mode SSH ingress isolation. When enabled, the driver creates a
|
||||
/// `NetworkPolicy` in each managed workspace namespace that permits TCP 2222
|
||||
/// only from gateway pods matching this peer.
|
||||
pub managed_ssh_ingress: ManagedSshIngressConfig,
|
||||
/// Image that provides the `openshell-sandbox` supervisor binary.
|
||||
/// Mounted directly as an image volume, or copied via an init container,
|
||||
/// depending on `supervisor_sideload_method`.
|
||||
/// Image that provides the trusted `openshell-sandbox` bootstrap binary
|
||||
/// and the separate `openshell-supervisor` control binary.
|
||||
pub supervisor_image: String,
|
||||
/// Pull policy for the supervisor image. Omit to use Kubernetes's image
|
||||
/// default.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub supervisor_image_pull_policy: Option<ImagePullPolicy>,
|
||||
/// How the supervisor binary is delivered into sandbox pods.
|
||||
pub supervisor_sideload_method: SupervisorSideloadMethod,
|
||||
/// How the supervisor is arranged for Kubernetes sandbox pods.
|
||||
pub topology: SupervisorTopology,
|
||||
/// Sidecar-only settings used when `topology = "sidecar"`.
|
||||
pub sidecar: KubernetesSidecarConfig,
|
||||
/// Kubernetes `imagePullPolicy` for the supervisor image.
|
||||
/// Empty string delegates to the Kubernetes default.
|
||||
pub supervisor_image_pull_policy: String,
|
||||
/// Cross-pod sandbox/supervisor settings.
|
||||
pub proxy_pod: KubernetesProxyPodConfig,
|
||||
/// Corporate HTTP forward proxy used by the network supervisor for
|
||||
/// policy-approved TLS CONNECT egress.
|
||||
pub https_proxy: Option<String>,
|
||||
@@ -278,14 +176,6 @@ pub struct KubernetesComputeConfig {
|
||||
pub client_tls_secret_name: String,
|
||||
pub host_gateway_ip: String,
|
||||
pub enable_user_namespaces: bool,
|
||||
/// Kubernetes `AppArmor` profile requested for the sandbox agent container.
|
||||
/// Empty/None omits the `appArmorProfile` field from sandbox pod specs.
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "deserialize_optional_app_armor_profile"
|
||||
)]
|
||||
pub app_armor_profile: Option<AppArmorProfile>,
|
||||
pub workspace_default_storage_size: String,
|
||||
/// Kubernetes `StorageClass` name for the default workspace PVC.
|
||||
/// Empty string (default) = omit `storageClassName`, using the cluster's
|
||||
@@ -314,13 +204,10 @@ pub struct KubernetesComputeConfig {
|
||||
deserialize_with = "deserialize_provider_spiffe_workload_api_socket_path"
|
||||
)]
|
||||
pub provider_spiffe_workload_api_socket_path: String,
|
||||
/// UID used for privilege-drop operations and workspace init container
|
||||
/// ownership. The supervisor container always runs as UID 0 (root) to
|
||||
/// create network namespaces and configure Landlock/seccomp; the
|
||||
/// `sandbox_uid` is injected as the `SANDBOX_UID` environment variable so
|
||||
/// the supervisor knows which UID to drop to for child processes.
|
||||
/// Exact UID shared by `openshell-sandbox`, its agent children, the trusted
|
||||
/// workspace/bootstrap init containers, and `openshell-supervisor`.
|
||||
/// When empty, the driver auto-detects from `OpenShift` SCC annotations on
|
||||
/// the target namespace; if those are also absent, falls back to `1000`.
|
||||
/// the target namespace; if those are also absent, falls back to `10001`.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub sandbox_uid: Option<u32>,
|
||||
/// GID used alongside `sandbox_uid` for PVC init container operations.
|
||||
@@ -347,7 +234,7 @@ pub const MAX_SA_TOKEN_TTL_SECS: i64 = 86_400;
|
||||
|
||||
/// Default sandbox UID used when neither config nor `OpenShift` SCC annotations
|
||||
/// provide a resolved value.
|
||||
pub(crate) const DEFAULT_SANDBOX_UID: u32 = 1000;
|
||||
pub(crate) const DEFAULT_SANDBOX_UID: u32 = 10001;
|
||||
|
||||
/// The annotation key for the `OpenShift` `ServiceAccount` UID range.
|
||||
/// Format: `<start>/<size>` (e.g. `1000000000/10000`).
|
||||
@@ -367,16 +254,16 @@ impl Default for KubernetesComputeConfig {
|
||||
operator_namespace_file: None,
|
||||
service_account_name: DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME.to_string(),
|
||||
default_image: openshell_core::image::default_sandbox_image(),
|
||||
// Omit the field so Kubernetes applies its own default (Always for
|
||||
// `latest`, IfNotPresent otherwise).
|
||||
image_pull_policy: None,
|
||||
// Default empty so the gateway omits `imagePullPolicy` from pod
|
||||
// specs and Kubernetes applies its own default (Always for `latest`,
|
||||
// IfNotPresent otherwise). `DEFAULT_IMAGE_PULL_POLICY` ("missing")
|
||||
// is Podman vocabulary and is not a valid Kubernetes value.
|
||||
image_pull_policy: String::new(),
|
||||
image_pull_secrets: Vec::new(),
|
||||
managed_ssh_ingress: ManagedSshIngressConfig::default(),
|
||||
supervisor_image: config::default_supervisor_image(),
|
||||
supervisor_image_pull_policy: None,
|
||||
supervisor_sideload_method: SupervisorSideloadMethod::default(),
|
||||
topology: SupervisorTopology::default(),
|
||||
sidecar: KubernetesSidecarConfig::default(),
|
||||
supervisor_image_pull_policy: String::new(),
|
||||
proxy_pod: KubernetesProxyPodConfig::default(),
|
||||
https_proxy: None,
|
||||
no_proxy: None,
|
||||
proxy_auth_secret_name: None,
|
||||
@@ -388,7 +275,6 @@ impl Default for KubernetesComputeConfig {
|
||||
client_tls_secret_name: String::new(),
|
||||
host_gateway_ip: String::new(),
|
||||
enable_user_namespaces: false,
|
||||
app_armor_profile: None,
|
||||
workspace_default_storage_size: DEFAULT_WORKSPACE_STORAGE_SIZE.to_string(),
|
||||
workspace_storage_class: String::new(),
|
||||
default_runtime_class_name: String::new(),
|
||||
@@ -401,16 +287,6 @@ impl Default for KubernetesComputeConfig {
|
||||
}
|
||||
|
||||
impl KubernetesComputeConfig {
|
||||
/// Validate startup configuration without connecting to Kubernetes.
|
||||
pub fn validate_configuration(&self) -> Result<(), String> {
|
||||
self.validate_workspace_mode()?;
|
||||
self.validate_provider_spiffe_workload_api_socket_path()?;
|
||||
self.validate_sandbox_identity_config()?;
|
||||
self.validate_proxy_uid()?;
|
||||
self.validate_image_pull_policies()?;
|
||||
self.validate_upstream_proxy_config()
|
||||
}
|
||||
|
||||
/// Clamp `sa_token_ttl_secs` into the `[MIN_SA_TOKEN_TTL_SECS,
|
||||
/// MAX_SA_TOKEN_TTL_SECS]` range used by the projected-volume spec.
|
||||
/// Invalid (≤0) values fall back to the default 3600.
|
||||
@@ -439,57 +315,20 @@ impl KubernetesComputeConfig {
|
||||
}
|
||||
|
||||
pub fn validate_proxy_uid(&self) -> Result<(), String> {
|
||||
self.sidecar.validate_proxy_uid()
|
||||
}
|
||||
|
||||
/// Reject pull policies Kubernetes cannot express before creating pods.
|
||||
pub fn validate_image_pull_policies(&self) -> Result<(), String> {
|
||||
for (field, policy) in [
|
||||
("image_pull_policy", self.image_pull_policy),
|
||||
(
|
||||
"supervisor_image_pull_policy",
|
||||
self.supervisor_image_pull_policy,
|
||||
),
|
||||
] {
|
||||
if policy == Some(ImagePullPolicy::Newer) {
|
||||
return Err(format!(
|
||||
"{field} = \"newer\" is supported only by the Podman compute driver"
|
||||
));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Translate a shared policy to Kubernetes's API vocabulary.
|
||||
pub fn image_pull_policy_value(policy: ImagePullPolicy) -> Result<&'static str, String> {
|
||||
match policy {
|
||||
ImagePullPolicy::Always => Ok("Always"),
|
||||
ImagePullPolicy::IfNotPresent => Ok("IfNotPresent"),
|
||||
ImagePullPolicy::Never => Ok("Never"),
|
||||
ImagePullPolicy::Newer => Err(
|
||||
"image pull policy 'newer' is supported only by the Podman compute driver"
|
||||
.to_string(),
|
||||
),
|
||||
}
|
||||
self.proxy_pod.validate()
|
||||
}
|
||||
|
||||
/// Validate the operator-owned corporate upstream proxy configuration.
|
||||
pub fn validate_upstream_proxy_config(&self) -> Result<(), String> {
|
||||
use openshell_core::driver_utils::{UpstreamProxyUrlError, parse_upstream_proxy_url};
|
||||
|
||||
let proxy_addr = self
|
||||
.https_proxy
|
||||
.as_deref()
|
||||
.map(|url| {
|
||||
parse_upstream_proxy_url(url).map_err(|err| match err {
|
||||
UpstreamProxyUrlError::Empty => {
|
||||
"https_proxy must not be empty when set".to_string()
|
||||
}
|
||||
UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(),
|
||||
err => format!("https_proxy {err}"),
|
||||
})
|
||||
})
|
||||
.transpose()?;
|
||||
if let Some(url) = &self.https_proxy {
|
||||
parse_upstream_proxy_url(url).map_err(|err| match err {
|
||||
UpstreamProxyUrlError::Empty => "https_proxy must not be empty when set".to_string(),
|
||||
UpstreamProxyUrlError::InlineCredentials => "https_proxy must not embed credentials in the URL; supply them through proxy_auth_secret_name and proxy_auth_secret_key".to_string(),
|
||||
err => format!("https_proxy {err}"),
|
||||
})?;
|
||||
}
|
||||
|
||||
if let Some(list) = self.no_proxy.as_deref() {
|
||||
if list.trim().is_empty() {
|
||||
@@ -548,17 +387,9 @@ impl KubernetesComputeConfig {
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
if proxy_addr.as_ref().is_some_and(|proxy| !proxy.secure)
|
||||
&& self.proxy_auth_allow_insecure != Some(true)
|
||||
{
|
||||
if self.proxy_auth_allow_insecure != Some(true) {
|
||||
return Err("proxy credentials use cleartext Basic auth over the connection to the http:// proxy; set proxy_auth_allow_insecure = true to accept that exposure, or remove the credential Secret".to_string());
|
||||
}
|
||||
if self.topology == SupervisorTopology::Combined {
|
||||
return Err(
|
||||
"proxy credential Secrets require topology = \"sidecar\"; combined topology shares the credential mount with the workload and fsGroup can make it readable by the sandbox user"
|
||||
.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
return Err(
|
||||
@@ -583,7 +414,7 @@ impl KubernetesComputeConfig {
|
||||
/// 2. `OpenShift` SCC namespace annotations (`sa.scc.uid-range`,
|
||||
/// `sa.scc.supplemental-groups`) — passed in as the optional
|
||||
/// `namespace_annotations` map
|
||||
/// 3. Fallback defaults: UID=`1000`, GID=UID
|
||||
/// 3. Fallback defaults: UID=`10001`, GID=UID
|
||||
pub fn resolve_sandbox_uid(
|
||||
&self,
|
||||
namespace_annotations: Option<&BTreeMap<String, String>>,
|
||||
@@ -885,87 +716,16 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_topology_is_combined() {
|
||||
let cfg = KubernetesComputeConfig::default();
|
||||
assert_eq!(cfg.topology, SupervisorTopology::Combined);
|
||||
assert_eq!(cfg.topology.to_string(), "combined");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_proxy_uid_is_dedicated_non_root_uid() {
|
||||
let cfg = KubernetesComputeConfig::default();
|
||||
assert_eq!(cfg.sidecar.proxy_uid, DEFAULT_PROXY_UID);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_sidecar_requires_process_binary_aware_network_policy() {
|
||||
let cfg = KubernetesComputeConfig::default();
|
||||
assert!(cfg.sidecar.process_binary_aware_network_policy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_pull_policy_uses_shared_canonical_values() {
|
||||
let cfg: KubernetesComputeConfig = serde_json::from_value(serde_json::json!({
|
||||
"image_pull_policy": "if_not_present",
|
||||
"supervisor_image_pull_policy": "never"
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(cfg.image_pull_policy, Some(ImagePullPolicy::IfNotPresent));
|
||||
assert_eq!(
|
||||
cfg.supervisor_image_pull_policy,
|
||||
Some(ImagePullPolicy::Never)
|
||||
);
|
||||
|
||||
for (policy, expected) in [
|
||||
(ImagePullPolicy::Always, "Always"),
|
||||
(ImagePullPolicy::IfNotPresent, "IfNotPresent"),
|
||||
(ImagePullPolicy::Never, "Never"),
|
||||
] {
|
||||
assert_eq!(
|
||||
KubernetesComputeConfig::image_pull_policy_value(policy).unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn image_pull_policy_rejects_newer_for_sandbox_and_supervisor_images() {
|
||||
fn proxy_pod_requires_network_policy_enforcement_acknowledgement() {
|
||||
let mut cfg = KubernetesComputeConfig::default();
|
||||
assert!(
|
||||
KubernetesComputeConfig::image_pull_policy_value(ImagePullPolicy::Newer)
|
||||
cfg.validate_proxy_uid()
|
||||
.unwrap_err()
|
||||
.contains("supported only by the Podman")
|
||||
.contains("network_policy_enforced")
|
||||
);
|
||||
|
||||
for (sandbox, supervisor) in [
|
||||
(Some(ImagePullPolicy::Newer), None),
|
||||
(None, Some(ImagePullPolicy::Newer)),
|
||||
] {
|
||||
let cfg = KubernetesComputeConfig {
|
||||
image_pull_policy: sandbox,
|
||||
supervisor_image_pull_policy: supervisor,
|
||||
..KubernetesComputeConfig::default()
|
||||
};
|
||||
let error = cfg.validate_image_pull_policies().unwrap_err();
|
||||
assert!(error.contains("supported only by the Podman"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_override_topology_sidecar() {
|
||||
let json = serde_json::json!({
|
||||
"topology": "sidecar"
|
||||
});
|
||||
let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(cfg.topology, SupervisorTopology::Sidecar);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_override_topology_combined() {
|
||||
let json = serde_json::json!({
|
||||
"topology": "combined"
|
||||
});
|
||||
let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(cfg.topology, SupervisorTopology::Combined);
|
||||
cfg.proxy_pod.network_policy_enforced = true;
|
||||
cfg.validate_proxy_uid().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -979,51 +739,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("unknown field"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_override_sidecar_process_binary_aware_network_policy_nested() {
|
||||
let json = serde_json::json!({
|
||||
"sidecar": {
|
||||
"process_binary_aware_network_policy": false
|
||||
}
|
||||
});
|
||||
let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap();
|
||||
assert!(!cfg.sidecar.process_binary_aware_network_policy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_override_sidecar_proxy_uid_nested() {
|
||||
let json = serde_json::json!({
|
||||
"sidecar": {
|
||||
"proxy_uid": 2000
|
||||
}
|
||||
});
|
||||
let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(cfg.sidecar.proxy_uid, 2000);
|
||||
cfg.validate_proxy_uid().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_proxy_uid_rejects_privileged_uid() {
|
||||
let cfg = KubernetesComputeConfig {
|
||||
sidecar: KubernetesSidecarConfig {
|
||||
proxy_uid: 999,
|
||||
..KubernetesSidecarConfig::default()
|
||||
},
|
||||
..KubernetesComputeConfig::default()
|
||||
};
|
||||
let err = cfg.validate_proxy_uid().unwrap_err();
|
||||
assert!(err.contains("proxy_uid"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_rejects_invalid_topology() {
|
||||
let json = serde_json::json!({
|
||||
"topology": "unsupported"
|
||||
});
|
||||
let err = serde_json::from_value::<KubernetesComputeConfig>(json).unwrap_err();
|
||||
assert!(err.to_string().contains("unknown variant"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_rejects_removed_topology_alias_field() {
|
||||
let mut json = serde_json::Map::new();
|
||||
@@ -1108,53 +823,6 @@ mod tests {
|
||||
assert!(cfg.default_runtime_class_name.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_app_armor_profile_is_none() {
|
||||
let cfg = KubernetesComputeConfig::default();
|
||||
assert!(cfg.app_armor_profile.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_override_app_armor_profile_unconfined() {
|
||||
let json = serde_json::json!({
|
||||
"app_armor_profile": "Unconfined"
|
||||
});
|
||||
let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(cfg.app_armor_profile, Some(AppArmorProfile::Unconfined));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_override_app_armor_profile_runtime_default() {
|
||||
let json = serde_json::json!({
|
||||
"app_armor_profile": "RuntimeDefault"
|
||||
});
|
||||
let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(cfg.app_armor_profile, Some(AppArmorProfile::RuntimeDefault));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_override_app_armor_profile_localhost() {
|
||||
let json = serde_json::json!({
|
||||
"app_armor_profile": "Localhost/openshell-supervisor"
|
||||
});
|
||||
let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(
|
||||
cfg.app_armor_profile,
|
||||
Some(AppArmorProfile::Localhost(
|
||||
"openshell-supervisor".to_string()
|
||||
))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_empty_app_armor_profile_disables_field() {
|
||||
let json = serde_json::json!({
|
||||
"app_armor_profile": ""
|
||||
});
|
||||
let cfg: KubernetesComputeConfig = serde_json::from_value(json).unwrap();
|
||||
assert_eq!(cfg.app_armor_profile, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_accepts_absolute_provider_spiffe_socket_path() {
|
||||
let json = serde_json::json!({
|
||||
@@ -1184,15 +852,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_rejects_invalid_app_armor_profile() {
|
||||
let json = serde_json::json!({
|
||||
"app_armor_profile": "runtime/default"
|
||||
});
|
||||
let err = serde_json::from_value::<KubernetesComputeConfig>(json).unwrap_err();
|
||||
assert!(err.to_string().contains("unknown AppArmor profile"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serde_override_image_pull_secrets() {
|
||||
let json = serde_json::json!({
|
||||
@@ -1399,7 +1058,6 @@ mod tests {
|
||||
#[test]
|
||||
fn upstream_proxy_config_accepts_secret_credentials_with_acknowledgement() {
|
||||
let cfg = KubernetesComputeConfig {
|
||||
topology: SupervisorTopology::Sidecar,
|
||||
https_proxy: Some("http://proxy.corp.example:8080".to_string()),
|
||||
proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()),
|
||||
proxy_auth_secret_key: Some("credentials".to_string()),
|
||||
@@ -1410,22 +1068,9 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_proxy_config_accepts_tls_protected_secret_credentials() {
|
||||
let cfg = KubernetesComputeConfig {
|
||||
topology: SupervisorTopology::Sidecar,
|
||||
https_proxy: Some("https://proxy.corp.example:8443".to_string()),
|
||||
proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()),
|
||||
proxy_auth_secret_key: Some("credentials".to_string()),
|
||||
..KubernetesComputeConfig::default()
|
||||
};
|
||||
assert!(cfg.validate_upstream_proxy_config().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn toml_deserializes_sidecar_upstream_proxy_settings() {
|
||||
fn toml_deserializes_upstream_proxy_settings() {
|
||||
let cfg: KubernetesComputeConfig = toml::from_str(
|
||||
r#"
|
||||
topology = "sidecar"
|
||||
https_proxy = "http://proxy.corp.example:8080"
|
||||
no_proxy = ".svc.cluster.local,10.96.0.0/12"
|
||||
proxy_auth_secret_name = "corporate-proxy-auth"
|
||||
@@ -1547,7 +1192,6 @@ mod tests {
|
||||
"bad key".to_string(), // whitespace is outside the allowed charset
|
||||
] {
|
||||
let cfg = KubernetesComputeConfig {
|
||||
topology: SupervisorTopology::Sidecar,
|
||||
https_proxy: Some("http://proxy.corp.example:8080".to_string()),
|
||||
proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()),
|
||||
proxy_auth_secret_key: Some(key.clone()),
|
||||
@@ -1565,7 +1209,6 @@ mod tests {
|
||||
#[test]
|
||||
fn upstream_proxy_config_accepts_max_length_secret_key() {
|
||||
let cfg = KubernetesComputeConfig {
|
||||
topology: SupervisorTopology::Sidecar,
|
||||
https_proxy: Some("http://proxy.corp.example:8080".to_string()),
|
||||
proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()),
|
||||
proxy_auth_secret_key: Some("a".repeat(253)),
|
||||
@@ -1575,20 +1218,6 @@ mod tests {
|
||||
assert!(cfg.validate_upstream_proxy_config().is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_proxy_config_rejects_credentials_in_combined_topology() {
|
||||
let cfg = KubernetesComputeConfig {
|
||||
topology: SupervisorTopology::Combined,
|
||||
https_proxy: Some("http://proxy.corp.example:8080".to_string()),
|
||||
proxy_auth_secret_name: Some("corporate-proxy-auth".to_string()),
|
||||
proxy_auth_secret_key: Some("credentials".to_string()),
|
||||
proxy_auth_allow_insecure: Some(true),
|
||||
..KubernetesComputeConfig::default()
|
||||
};
|
||||
let err = cfg.validate_upstream_proxy_config().unwrap_err();
|
||||
assert!(err.contains("topology = \"sidecar\""), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upstream_proxy_config_allows_explicit_false_acknowledgement_without_credentials() {
|
||||
let cfg = KubernetesComputeConfig {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -158,19 +158,21 @@ impl ComputeDriver for ComputeDriverService {
|
||||
&self,
|
||||
request: Request<CreateSandboxRequest>,
|
||||
) -> Result<Response<CreateSandboxResponse>, Status> {
|
||||
self.rpc_tracer
|
||||
.trace(openshell_otel::rpc::CREATE_SANDBOX, async {
|
||||
let sandbox = request
|
||||
.into_inner()
|
||||
.sandbox
|
||||
.ok_or_else(|| Status::invalid_argument("sandbox is required"))?;
|
||||
self.driver
|
||||
.create_sandbox(&sandbox)
|
||||
.await
|
||||
.map_err(|e| Status::from(openshell_core::ComputeDriverError::from(e)))?;
|
||||
Ok(Response::new(CreateSandboxResponse {}))
|
||||
})
|
||||
.await
|
||||
Box::pin(
|
||||
self.rpc_tracer
|
||||
.trace(openshell_otel::rpc::CREATE_SANDBOX, async {
|
||||
let sandbox = request
|
||||
.into_inner()
|
||||
.sandbox
|
||||
.ok_or_else(|| Status::invalid_argument("sandbox is required"))?;
|
||||
self.driver
|
||||
.create_sandbox(&sandbox)
|
||||
.await
|
||||
.map_err(|e| Status::from(openshell_core::ComputeDriverError::from(e)))?;
|
||||
Ok(Response::new(CreateSandboxResponse {}))
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn stop_sandbox(
|
||||
@@ -198,21 +200,25 @@ impl ComputeDriver for ComputeDriverService {
|
||||
&self,
|
||||
request: Request<StartSandboxRequest>,
|
||||
) -> Result<Response<StartSandboxResponse>, Status> {
|
||||
self.rpc_tracer
|
||||
.trace(openshell_otel::rpc::START_SANDBOX, async {
|
||||
let request = request.into_inner();
|
||||
if request.sandbox_id.is_empty() {
|
||||
return Err(Status::invalid_argument("sandbox_id is required"));
|
||||
}
|
||||
self.driver
|
||||
.start_sandbox(&request.sandbox_id)
|
||||
Box::pin(
|
||||
self.rpc_tracer
|
||||
.trace(openshell_otel::rpc::START_SANDBOX, async {
|
||||
let request = request.into_inner();
|
||||
if request.sandbox_id.is_empty() {
|
||||
return Err(Status::invalid_argument("sandbox_id is required"));
|
||||
}
|
||||
Box::pin(
|
||||
self.driver
|
||||
.start_sandbox(&request.sandbox_id, &request.launch_authentication),
|
||||
)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
Status::from(openshell_core::ComputeDriverError::from(error))
|
||||
})?;
|
||||
Ok(Response::new(StartSandboxResponse {}))
|
||||
})
|
||||
.await
|
||||
Ok(Response::new(StartSandboxResponse {}))
|
||||
}),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn delete_sandbox(
|
||||
|
||||
@@ -0,0 +1,396 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Kubernetes provisioning for the shared authenticated boundary protocol.
|
||||
//!
|
||||
//! This module deliberately contains no lifecycle, process, network, identity,
|
||||
//! or wire implementation. The driver chooses the proxy-pod placement, binds
|
||||
//! immutable Kubernetes resource identities, and provisions TCP coordinates;
|
||||
//! `openshell-isolation-interface` and `openshell-sandbox` provide the common
|
||||
//! control and boundary behavior.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::net::{IpAddr, SocketAddr};
|
||||
use std::path::PathBuf;
|
||||
|
||||
use k8s_openapi::api::networking::v1::{
|
||||
NetworkPolicy, NetworkPolicyIngressRule, NetworkPolicyPeer, NetworkPolicyPort,
|
||||
NetworkPolicySpec,
|
||||
};
|
||||
use k8s_openapi::apimachinery::pkg::apis::meta::v1::LabelSelector;
|
||||
use k8s_openapi::apimachinery::pkg::util::intstr::IntOrString;
|
||||
use kube::core::ObjectMeta;
|
||||
use openshell_isolation_interface::boundary_protocol::{
|
||||
BoundaryConfig, BoundaryListener, BoundaryTopology, GatewayVerificationKey,
|
||||
SandboxTlsClientConfig, SandboxTlsServerConfig, SandboxTransport,
|
||||
};
|
||||
use openshell_isolation_interface::contract::{DriverFenceEvidence, ResolvedWorkloadIdentity};
|
||||
|
||||
/// Registered RFC 0012 backend name for the proxy-pod topology.
|
||||
pub const BACKEND_NAME: &str = "kubernetes-proxy-pod";
|
||||
|
||||
/// Label that binds the workload and control pods in one unique pair.
|
||||
pub const BOUNDARY_PAIR_LABEL: &str = "openshell.ai/boundary-pair";
|
||||
|
||||
/// Label distinguishing the two pods in a boundary pair.
|
||||
pub const BOUNDARY_ROLE_LABEL: &str = "openshell.ai/boundary-role";
|
||||
|
||||
const WORKLOAD_ROLE: &str = "workload";
|
||||
const SUPERVISOR_ROLE: &str = "supervisor";
|
||||
|
||||
/// Driver-owned inputs for the workload pod's Kubernetes network fence.
|
||||
///
|
||||
/// This is the first phase of proxy-pod provisioning. The driver applies the
|
||||
/// returned labels to the respective pods and creates the returned policy. It
|
||||
/// then observes the policy UID and resourceVersion and supplies both to
|
||||
/// `KubernetesProxyPodBoundarySpec`.
|
||||
pub struct KubernetesProxyPodNetworkFenceSpec {
|
||||
pub namespace: String,
|
||||
pub policy_name: String,
|
||||
/// A unique, Kubernetes-label-safe value generated for this pod pair.
|
||||
pub pair_label_value: String,
|
||||
pub boundary_port: u16,
|
||||
}
|
||||
|
||||
/// Labels and policy needed to remove direct workload-pod egress.
|
||||
pub struct KubernetesProxyPodNetworkFence {
|
||||
pub workload_labels: BTreeMap<String, String>,
|
||||
pub control_labels: BTreeMap<String, String>,
|
||||
pub workload_policy: NetworkPolicy,
|
||||
}
|
||||
|
||||
impl KubernetesProxyPodNetworkFenceSpec {
|
||||
/// Render a default-deny workload fence with one control-to-boundary path.
|
||||
///
|
||||
/// Kubernetes `NetworkPolicy` is connection-aware: traffic returning over
|
||||
/// the control-initiated boundary connection is allowed even though the
|
||||
/// workload pod has no egress rules. The control pod remains responsible
|
||||
/// for opening policy-approved upstream connections.
|
||||
#[must_use]
|
||||
pub fn provision(self) -> KubernetesProxyPodNetworkFence {
|
||||
let workload_labels = boundary_pair_labels(&self.pair_label_value, WORKLOAD_ROLE);
|
||||
let control_labels = boundary_pair_labels(&self.pair_label_value, SUPERVISOR_ROLE);
|
||||
|
||||
let workload_policy = NetworkPolicy {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(self.policy_name),
|
||||
namespace: Some(self.namespace),
|
||||
..Default::default()
|
||||
},
|
||||
spec: Some(NetworkPolicySpec {
|
||||
pod_selector: LabelSelector {
|
||||
match_labels: Some(workload_labels.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
policy_types: Some(vec!["Ingress".to_string(), "Egress".to_string()]),
|
||||
// Only the exactly paired control pod in this namespace may
|
||||
// establish the authenticated boundary connection.
|
||||
ingress: Some(vec![NetworkPolicyIngressRule {
|
||||
from: Some(vec![NetworkPolicyPeer {
|
||||
pod_selector: Some(LabelSelector {
|
||||
match_labels: Some(control_labels.clone()),
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
}]),
|
||||
ports: Some(vec![NetworkPolicyPort {
|
||||
port: Some(IntOrString::Int(i32::from(self.boundary_port))),
|
||||
protocol: Some("TCP".to_string()),
|
||||
..Default::default()
|
||||
}]),
|
||||
}]),
|
||||
// An explicit empty list selects the pod for egress and allows
|
||||
// no new workload-initiated connections, including DNS and the
|
||||
// Kubernetes API. Reply traffic for allowed ingress remains
|
||||
// permitted by conforming NetworkPolicy implementations.
|
||||
egress: Some(Vec::new()),
|
||||
}),
|
||||
status: None,
|
||||
};
|
||||
|
||||
KubernetesProxyPodNetworkFence {
|
||||
workload_labels,
|
||||
control_labels,
|
||||
workload_policy,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn boundary_pair_labels(pair: &str, role: &str) -> BTreeMap<String, String> {
|
||||
BTreeMap::from([
|
||||
(BOUNDARY_PAIR_LABEL.to_string(), pair.to_string()),
|
||||
(BOUNDARY_ROLE_LABEL.to_string(), role.to_string()),
|
||||
])
|
||||
}
|
||||
|
||||
/// Driver-owned inputs that bind one workload/proxy pair to one boundary.
|
||||
///
|
||||
/// The driver constructs this only after Kubernetes has assigned every UID and
|
||||
/// after it has observed the exact egress policy resource version. The workload
|
||||
/// stays held until the matching boundary config and control topology have been
|
||||
/// installed.
|
||||
pub struct KubernetesProxyPodBoundarySpec {
|
||||
pub boundary_id: String,
|
||||
pub generation: String,
|
||||
pub session_id: openshell_core::SandboxSessionId,
|
||||
pub gateway_id: String,
|
||||
pub verification_keys: Vec<GatewayVerificationKey>,
|
||||
pub namespace_uid: String,
|
||||
pub sandbox_resource_uid: String,
|
||||
pub workload_pod_uid: String,
|
||||
pub workload_pod_uid_path: PathBuf,
|
||||
pub control_deployment_uid: String,
|
||||
pub egress_policy_uid: String,
|
||||
pub egress_policy_resource_version: String,
|
||||
pub boundary_listener: SocketAddr,
|
||||
pub control_address: SocketAddr,
|
||||
pub sandbox_tls: SandboxTlsServerConfig,
|
||||
pub supervisor_tls: SandboxTlsClientConfig,
|
||||
pub host_gateway_ip: Option<IpAddr>,
|
||||
pub workload_identity: ResolvedWorkloadIdentity,
|
||||
pub child_env: HashMap<String, String>,
|
||||
}
|
||||
|
||||
/// Protected workload-pod config and matching proxy-pod descriptor.
|
||||
pub struct KubernetesProxyPodBoundaryProvisioning {
|
||||
pub boundary_config: BoundaryConfig,
|
||||
pub topology: BoundaryTopology,
|
||||
}
|
||||
|
||||
impl KubernetesProxyPodBoundarySpec {
|
||||
/// Produce both sides of the common protocol from one observed Kubernetes
|
||||
/// resource set so a stale or recreated object cannot be attached.
|
||||
#[must_use]
|
||||
pub fn provision(self) -> KubernetesProxyPodBoundaryProvisioning {
|
||||
let resource_claims = BTreeMap::from([
|
||||
("kubernetes.namespace_uid".to_string(), self.namespace_uid),
|
||||
(
|
||||
"kubernetes.sandbox_resource_uid".to_string(),
|
||||
self.sandbox_resource_uid,
|
||||
),
|
||||
(
|
||||
"kubernetes.workload_pod_uid".to_string(),
|
||||
self.workload_pod_uid,
|
||||
),
|
||||
(
|
||||
"kubernetes.control_deployment_uid".to_string(),
|
||||
self.control_deployment_uid,
|
||||
),
|
||||
(
|
||||
"kubernetes.egress_policy_uid".to_string(),
|
||||
self.egress_policy_uid,
|
||||
),
|
||||
(
|
||||
"kubernetes.egress_policy_resource_version".to_string(),
|
||||
self.egress_policy_resource_version,
|
||||
),
|
||||
]);
|
||||
let driver_fence = DriverFenceEvidence::Kubernetes {
|
||||
network_policy_uid: resource_claims["kubernetes.egress_policy_uid"].clone(),
|
||||
network_policy_resource_version:
|
||||
resource_claims["kubernetes.egress_policy_resource_version"].clone(),
|
||||
ingress_isolated: true,
|
||||
egress_isolated: true,
|
||||
egress_rule_count: 0,
|
||||
};
|
||||
KubernetesProxyPodBoundaryProvisioning {
|
||||
boundary_config: BoundaryConfig {
|
||||
boundary_id: self.boundary_id.clone(),
|
||||
generation: self.generation.clone(),
|
||||
session_id: self.session_id,
|
||||
gateway_id: self.gateway_id,
|
||||
verification_keys: self.verification_keys,
|
||||
listener: BoundaryListener::TlsTcp {
|
||||
address: self.boundary_listener,
|
||||
tls: self.sandbox_tls,
|
||||
},
|
||||
resource_claims: resource_claims.clone(),
|
||||
resource_claim_files: BTreeMap::from([(
|
||||
"kubernetes.workload_pod_uid".to_string(),
|
||||
self.workload_pod_uid_path,
|
||||
)]),
|
||||
workload_identity: self.workload_identity.clone(),
|
||||
driver_fence: driver_fence.clone(),
|
||||
child_env: self.child_env,
|
||||
},
|
||||
topology: BoundaryTopology {
|
||||
boundary_id: self.boundary_id,
|
||||
generation: self.generation,
|
||||
session_id: self.session_id,
|
||||
workload_identity: self.workload_identity,
|
||||
transport: SandboxTransport::Tcp {
|
||||
authority: self.control_address.to_string(),
|
||||
addresses: vec![self.control_address],
|
||||
},
|
||||
tls: self.supervisor_tls,
|
||||
host_gateway_ip: self.host_gateway_ip,
|
||||
resource_claims,
|
||||
driver_fence,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn spec() -> KubernetesProxyPodBoundarySpec {
|
||||
KubernetesProxyPodBoundarySpec {
|
||||
boundary_id: "sandbox-1".to_string(),
|
||||
generation: "generation-1".to_string(),
|
||||
session_id: openshell_core::SandboxSessionId::new(),
|
||||
gateway_id: "gateway-1".to_string(),
|
||||
verification_keys: vec![GatewayVerificationKey {
|
||||
key_id: "key-1".to_string(),
|
||||
public_key_pem: "public-key".to_string(),
|
||||
}],
|
||||
namespace_uid: "namespace-uid".to_string(),
|
||||
sandbox_resource_uid: "sandbox-resource-uid".to_string(),
|
||||
workload_pod_uid: "pod-uid".to_string(),
|
||||
workload_pod_uid_path: PathBuf::from("/.openshell/pod-identity/uid"),
|
||||
control_deployment_uid: "control-deployment-uid".to_string(),
|
||||
egress_policy_uid: "network-policy-uid".to_string(),
|
||||
egress_policy_resource_version: "1945".to_string(),
|
||||
boundary_listener: "0.0.0.0:5500".parse().expect("valid listener"),
|
||||
control_address: "10.42.0.7:5500".parse().expect("valid target"),
|
||||
sandbox_tls: SandboxTlsServerConfig {
|
||||
certificate_chain_path: PathBuf::from("/run/boundary/tls.crt"),
|
||||
private_key_path: PathBuf::from("/run/boundary/tls.key"),
|
||||
},
|
||||
supervisor_tls: SandboxTlsClientConfig {
|
||||
server_name: "boundary.sandbox.openshell".to_string(),
|
||||
trust_anchor_pem: "test-ca".to_string(),
|
||||
},
|
||||
host_gateway_ip: Some("10.42.0.1".parse().expect("valid gateway IP")),
|
||||
workload_identity: ResolvedWorkloadIdentity::new(
|
||||
1000,
|
||||
1000,
|
||||
vec![1000],
|
||||
"kubernetes-config".to_string(),
|
||||
"sandbox:sandbox-resource-uid".to_string(),
|
||||
)
|
||||
.unwrap(),
|
||||
child_env: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provisioning_binds_identical_kubernetes_resource_claims() {
|
||||
let provisioned = spec().provision();
|
||||
|
||||
assert_eq!(
|
||||
provisioned.boundary_config.resource_claims,
|
||||
provisioned.topology.resource_claims
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.topology.resource_claims["kubernetes.sandbox_resource_uid"],
|
||||
"sandbox-resource-uid"
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.topology.resource_claims["kubernetes.egress_policy_resource_version"],
|
||||
"1945"
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.boundary_config.driver_fence,
|
||||
provisioned.topology.driver_fence
|
||||
);
|
||||
assert!(
|
||||
provisioned
|
||||
.topology
|
||||
.driver_fence
|
||||
.validate_for_backend(BACKEND_NAME)
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provisioning_uses_one_shared_tcp_protocol_across_pods() {
|
||||
let provisioned = spec().provision();
|
||||
|
||||
assert_eq!(
|
||||
provisioned.boundary_config.listener,
|
||||
BoundaryListener::TlsTcp {
|
||||
address: "0.0.0.0:5500".parse().expect("valid listener"),
|
||||
tls: SandboxTlsServerConfig {
|
||||
certificate_chain_path: PathBuf::from("/run/boundary/tls.crt"),
|
||||
private_key_path: PathBuf::from("/run/boundary/tls.key"),
|
||||
},
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.topology.transport,
|
||||
SandboxTransport::Tcp {
|
||||
authority: "10.42.0.7:5500".to_string(),
|
||||
addresses: vec!["10.42.0.7:5500".parse().expect("valid target")],
|
||||
}
|
||||
);
|
||||
assert_eq!(
|
||||
provisioned.topology.tls,
|
||||
SandboxTlsClientConfig {
|
||||
server_name: "boundary.sandbox.openshell".to_string(),
|
||||
trust_anchor_pem: "test-ca".to_string(),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_fence_denies_all_workload_initiated_egress() {
|
||||
let fence = KubernetesProxyPodNetworkFenceSpec {
|
||||
namespace: "sandbox-ns".to_string(),
|
||||
policy_name: "openshell-boundary-sandbox-1".to_string(),
|
||||
pair_label_value: "pair-1".to_string(),
|
||||
boundary_port: 5500,
|
||||
}
|
||||
.provision();
|
||||
|
||||
let policy_spec = fence.workload_policy.spec.expect("policy has a spec");
|
||||
assert_eq!(
|
||||
policy_spec.policy_types,
|
||||
Some(vec!["Ingress".to_string(), "Egress".to_string()])
|
||||
);
|
||||
assert_eq!(policy_spec.egress, Some(Vec::new()));
|
||||
assert_eq!(
|
||||
policy_spec.pod_selector.match_labels,
|
||||
Some(fence.workload_labels)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn network_fence_allows_only_paired_control_to_boundary_port() {
|
||||
let fence = KubernetesProxyPodNetworkFenceSpec {
|
||||
namespace: "sandbox-ns".to_string(),
|
||||
policy_name: "openshell-boundary-sandbox-1".to_string(),
|
||||
pair_label_value: "pair-1".to_string(),
|
||||
boundary_port: 5500,
|
||||
}
|
||||
.provision();
|
||||
|
||||
let policy_spec = fence.workload_policy.spec.expect("policy has a spec");
|
||||
let ingress = policy_spec
|
||||
.ingress
|
||||
.expect("policy has ingress rules")
|
||||
.pop()
|
||||
.expect("policy has one ingress rule");
|
||||
let peer = ingress
|
||||
.from
|
||||
.expect("rule has peers")
|
||||
.pop()
|
||||
.expect("rule has one peer");
|
||||
assert_eq!(
|
||||
peer.pod_selector
|
||||
.expect("peer has a pod selector")
|
||||
.match_labels,
|
||||
Some(fence.control_labels)
|
||||
);
|
||||
assert!(peer.namespace_selector.is_none());
|
||||
|
||||
let port = ingress
|
||||
.ports
|
||||
.expect("rule has ports")
|
||||
.pop()
|
||||
.expect("rule has one port");
|
||||
assert_eq!(port.protocol.as_deref(), Some("TCP"));
|
||||
assert_eq!(port.port, Some(IntOrString::Int(5500)));
|
||||
}
|
||||
}
|
||||
@@ -4,12 +4,13 @@
|
||||
pub mod config;
|
||||
pub mod driver;
|
||||
pub mod grpc;
|
||||
pub mod isolation;
|
||||
pub mod otel_tracing;
|
||||
mod proxy_pod;
|
||||
|
||||
pub use config::{
|
||||
AppArmorProfile, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME,
|
||||
DEFAULT_WORKSPACE_STORAGE_SIZE, KubernetesComputeConfig, KubernetesSidecarConfig,
|
||||
ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology, WorkspaceMode,
|
||||
DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE,
|
||||
KubernetesComputeConfig, KubernetesProxyPodConfig, ManagedSshIngressConfig, WorkspaceMode,
|
||||
managed_namespace_prefix,
|
||||
};
|
||||
pub use driver::{KubernetesComputeDriver, KubernetesDriverError};
|
||||
|
||||
@@ -8,13 +8,12 @@ use std::net::SocketAddr;
|
||||
use std::path::PathBuf;
|
||||
use tracing::info;
|
||||
|
||||
use openshell_core::VERSION;
|
||||
use openshell_core::proto::compute::v1::compute_driver_server::ComputeDriverServer;
|
||||
use openshell_core::{ImagePullPolicy, VERSION};
|
||||
use openshell_driver_kubernetes::{
|
||||
AppArmorProfile, ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_PROXY_UID,
|
||||
DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, KubernetesComputeConfig, KubernetesComputeDriver,
|
||||
KubernetesSidecarConfig, ManagedSshIngressConfig, SupervisorSideloadMethod, SupervisorTopology,
|
||||
WorkspaceMode,
|
||||
ComputeDriverService, DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME,
|
||||
KubernetesComputeConfig, KubernetesComputeDriver, KubernetesProxyPodConfig,
|
||||
ManagedSshIngressConfig, WorkspaceMode,
|
||||
};
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
@@ -72,7 +71,7 @@ struct Args {
|
||||
sandbox_image: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SANDBOX_IMAGE_PULL_POLICY")]
|
||||
sandbox_image_pull_policy: Option<ImagePullPolicy>,
|
||||
sandbox_image_pull_policy: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
@@ -94,10 +93,8 @@ struct Args {
|
||||
)]
|
||||
managed_ssh_gateway_pod_selector: Vec<String>,
|
||||
|
||||
/// Gateway callback endpoint reachable from sandbox pods. Kubernetes
|
||||
/// service topology cannot be inferred from the sandbox namespace.
|
||||
#[arg(long, env = "OPENSHELL_GRPC_ENDPOINT")]
|
||||
grpc_endpoint: String,
|
||||
grpc_endpoint: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
@@ -116,33 +113,21 @@ struct Args {
|
||||
supervisor_image: Option<String>,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY")]
|
||||
supervisor_image_pull_policy: Option<ImagePullPolicy>,
|
||||
supervisor_image_pull_policy: Option<String>,
|
||||
|
||||
#[arg(
|
||||
long,
|
||||
env = "OPENSHELL_SUPERVISOR_SIDELOAD_METHOD",
|
||||
default_value = "image-volume"
|
||||
env = "OPENSHELL_K8S_PROXY_POD_NETWORK_POLICY_ENFORCED",
|
||||
default_value_t = false
|
||||
)]
|
||||
supervisor_sideload_method: SupervisorSideloadMethod,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_K8S_TOPOLOGY", default_value = "combined")]
|
||||
topology: SupervisorTopology,
|
||||
proxy_pod_network_policy_enforced: bool,
|
||||
|
||||
#[arg(
|
||||
long = "sidecar-proxy-uid",
|
||||
alias = "proxy-uid",
|
||||
env = "OPENSHELL_K8S_SIDECAR_PROXY_UID",
|
||||
default_value_t = DEFAULT_PROXY_UID
|
||||
long,
|
||||
env = "OPENSHELL_K8S_PROXY_POD_BOUNDARY_PORT",
|
||||
default_value_t = 5500
|
||||
)]
|
||||
sidecar_proxy_uid: u32,
|
||||
|
||||
#[arg(
|
||||
long = "sidecar-process-binary-aware-network-policy",
|
||||
env = "OPENSHELL_K8S_SIDECAR_PROCESS_BINARY_AWARE_NETWORK_POLICY",
|
||||
default_value_t = true,
|
||||
action = ArgAction::Set
|
||||
)]
|
||||
sidecar_process_binary_aware_network_policy: bool,
|
||||
proxy_pod_boundary_port: u16,
|
||||
|
||||
/// Corporate HTTP forward proxy for policy-approved TLS CONNECT egress.
|
||||
#[arg(long, env = "OPENSHELL_UPSTREAM_PROXY")]
|
||||
@@ -171,9 +156,6 @@ struct Args {
|
||||
#[arg(long, env = "OPENSHELL_ENABLE_USER_NAMESPACES")]
|
||||
enable_user_namespaces: bool,
|
||||
|
||||
#[arg(long, env = "OPENSHELL_K8S_APP_ARMOR_PROFILE")]
|
||||
app_armor_profile: Option<AppArmorProfile>,
|
||||
|
||||
/// Lifetime (seconds) of the projected `ServiceAccount` token
|
||||
/// kubelet writes into each sandbox pod for the `IssueSandboxToken`
|
||||
/// bootstrap exchange. Kubelet enforces a minimum of 600s; the
|
||||
@@ -250,7 +232,7 @@ async fn main() -> Result<()> {
|
||||
operator_namespace_file: args.operator_namespace_file,
|
||||
service_account_name: args.sandbox_service_account,
|
||||
default_image: args.sandbox_image.unwrap_or_default(),
|
||||
image_pull_policy: args.sandbox_image_pull_policy,
|
||||
image_pull_policy: args.sandbox_image_pull_policy.unwrap_or_default(),
|
||||
image_pull_secrets: args.sandbox_image_pull_secrets,
|
||||
managed_ssh_ingress: ManagedSshIngressConfig {
|
||||
enabled: args.managed_ssh_ingress_enabled,
|
||||
@@ -260,13 +242,10 @@ async fn main() -> Result<()> {
|
||||
supervisor_image: args
|
||||
.supervisor_image
|
||||
.unwrap_or_else(openshell_core::config::default_supervisor_image),
|
||||
supervisor_image_pull_policy: args.supervisor_image_pull_policy,
|
||||
supervisor_sideload_method: args.supervisor_sideload_method,
|
||||
topology: args.topology,
|
||||
sidecar: KubernetesSidecarConfig {
|
||||
proxy_uid: args.sidecar_proxy_uid,
|
||||
process_binary_aware_network_policy: args
|
||||
.sidecar_process_binary_aware_network_policy,
|
||||
supervisor_image_pull_policy: args.supervisor_image_pull_policy.unwrap_or_default(),
|
||||
proxy_pod: KubernetesProxyPodConfig {
|
||||
network_policy_enforced: args.proxy_pod_network_policy_enforced,
|
||||
boundary_port: args.proxy_pod_boundary_port,
|
||||
},
|
||||
https_proxy: args.https_proxy,
|
||||
no_proxy: args.no_proxy,
|
||||
@@ -274,12 +253,11 @@ async fn main() -> Result<()> {
|
||||
proxy_auth_secret_key: args.proxy_auth_secret_key,
|
||||
proxy_auth_allow_insecure: args.proxy_auth_allow_insecure.then_some(true),
|
||||
proxy_connect_by_hostname: args.proxy_connect_by_hostname.then_some(true),
|
||||
grpc_endpoint: args.grpc_endpoint,
|
||||
grpc_endpoint: args.grpc_endpoint.unwrap_or_default(),
|
||||
ssh_socket_path: args.sandbox_ssh_socket_path,
|
||||
client_tls_secret_name: args.client_tls_secret_name.unwrap_or_default(),
|
||||
host_gateway_ip: args.host_gateway_ip.unwrap_or_default(),
|
||||
enable_user_namespaces: args.enable_user_namespaces,
|
||||
app_armor_profile: args.app_armor_profile,
|
||||
workspace_default_storage_size: std::env::var(
|
||||
"OPENSHELL_K8S_WORKSPACE_DEFAULT_STORAGE_SIZE",
|
||||
)
|
||||
@@ -337,13 +315,6 @@ async fn main() -> Result<()> {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn requires_explicit_gateway_callback_endpoint() {
|
||||
let error = Args::try_parse_from(["openshell-driver-kubernetes"])
|
||||
.expect_err("Kubernetes service topology must be explicit");
|
||||
assert!(error.to_string().contains("--grpc-endpoint"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn accepts_gateway_otlp_configuration() {
|
||||
let args = Args::try_parse_from([
|
||||
@@ -352,8 +323,6 @@ mod tests {
|
||||
"http://collector.example:4317",
|
||||
"--gateway-name",
|
||||
"kubernetes-dev",
|
||||
"--grpc-endpoint",
|
||||
"http://openshell.example:8080",
|
||||
])
|
||||
.expect("OTLP endpoint should parse");
|
||||
|
||||
|
||||
@@ -0,0 +1,739 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Kubernetes resources owned by the RFC 0012 proxy-pod topology.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
use k8s_openapi::ByteString;
|
||||
use k8s_openapi::api::apps::v1::Deployment;
|
||||
use k8s_openapi::api::core::v1::{Secret, Service};
|
||||
use k8s_openapi::api::networking::v1::{NetworkPolicy, NetworkPolicyEgressRule, NetworkPolicySpec};
|
||||
use k8s_openapi::apimachinery::pkg::apis::meta::v1::{LabelSelector, OwnerReference};
|
||||
use kube::core::ObjectMeta;
|
||||
use rcgen::{CertificateParams, DnType, IsCa, KeyPair, KeyUsagePurpose};
|
||||
|
||||
use crate::isolation::{
|
||||
BOUNDARY_PAIR_LABEL, BOUNDARY_ROLE_LABEL, KubernetesProxyPodNetworkFence,
|
||||
KubernetesProxyPodNetworkFenceSpec,
|
||||
};
|
||||
|
||||
pub const SANDBOX_SECRET_COMPONENT: &str = "sandbox-bootstrap";
|
||||
pub const SUPERVISOR_SECRET_COMPONENT: &str = "supervisor-bootstrap";
|
||||
pub const BOUNDARY_CONFIG_KEY: &str = "boundary.json";
|
||||
pub const TOPOLOGY_PAYLOAD_KEY: &str = "topology.json";
|
||||
pub const BOUNDARY_CERTIFICATE_KEY: &str = "tls.crt";
|
||||
pub const BOUNDARY_PRIVATE_KEY: &str = "tls.key";
|
||||
pub const SUPERVISOR_AUTH_BUNDLE_KEY: &str = "auth.json";
|
||||
pub const PROXY_CA_CERTIFICATE_KEY: &str = "proxy-ca.crt";
|
||||
pub const PROXY_CA_PRIVATE_KEY: &str = "proxy-ca.key";
|
||||
pub const SANDBOX_BOOTSTRAP_INPUT_PATH: &str = "/.openshell/bootstrap-input";
|
||||
pub const BOUNDARY_CONFIG_PATH: &str = "/.openshell/state/bootstrap/boundary.json";
|
||||
pub const BOUNDARY_CERTIFICATE_PATH: &str = "/.openshell/state/bootstrap/tls.crt";
|
||||
pub const BOUNDARY_PRIVATE_KEY_PATH: &str = "/.openshell/state/bootstrap/tls.key";
|
||||
pub const TOPOLOGY_PAYLOAD_PATH: &str = "/.openshell/supervisor/topology.json";
|
||||
pub const SUPERVISOR_AUTH_BUNDLE_PATH: &str = "/.openshell/supervisor/auth.json";
|
||||
pub const PROXY_CA_CERTIFICATE_PATH: &str = "/.openshell/supervisor/proxy-ca.crt";
|
||||
pub const PROXY_CA_PRIVATE_KEY_PATH: &str = "/.openshell/supervisor/proxy-ca.key";
|
||||
pub const CONTROL_HEALTH_SOCKET_PATH: &str = "/run/openshell/health.sock";
|
||||
|
||||
pub struct ProxyCaMaterial {
|
||||
pub certificate_pem: String,
|
||||
pub private_key_pem: String,
|
||||
}
|
||||
|
||||
pub fn generate_proxy_ca_material() -> Result<ProxyCaMaterial, String> {
|
||||
let key = KeyPair::generate().map_err(|error| format!("generate proxy CA key: {error}"))?;
|
||||
let mut params = CertificateParams::default();
|
||||
params.is_ca = IsCa::Ca(rcgen::BasicConstraints::Unconstrained);
|
||||
params
|
||||
.distinguished_name
|
||||
.push(DnType::CommonName, "OpenShell Sandbox CA");
|
||||
params
|
||||
.distinguished_name
|
||||
.push(DnType::OrganizationName, "OpenShell");
|
||||
params.key_usages = vec![KeyUsagePurpose::KeyCertSign, KeyUsagePurpose::CrlSign];
|
||||
let certificate = params
|
||||
.self_signed(&key)
|
||||
.map_err(|error| format!("generate proxy CA certificate: {error}"))?;
|
||||
Ok(ProxyCaMaterial {
|
||||
certificate_pem: certificate.pem(),
|
||||
private_key_pem: key.serialize_pem(),
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ProxyPodNames {
|
||||
pub sandbox_secret: String,
|
||||
pub supervisor_secret: String,
|
||||
pub boundary_service: String,
|
||||
pub control_deployment: String,
|
||||
pub workload_policy: String,
|
||||
pub control_policy: String,
|
||||
}
|
||||
|
||||
impl ProxyPodNames {
|
||||
#[must_use]
|
||||
pub fn new(sandbox_id: &str) -> Self {
|
||||
let suffix = sandbox_id.to_ascii_lowercase();
|
||||
Self {
|
||||
sandbox_secret: format!("os-sandbox-{suffix}"),
|
||||
supervisor_secret: format!("os-supervisor-{suffix}"),
|
||||
boundary_service: format!("os-boundary-{suffix}"),
|
||||
control_deployment: format!("os-supervisor-{suffix}"),
|
||||
workload_policy: format!("os-boundary-{suffix}"),
|
||||
control_policy: format!("os-supervisor-{suffix}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Return stable companion names plus generation-specific immutable
|
||||
/// bootstrap Secret names.
|
||||
#[must_use]
|
||||
pub fn for_generation(sandbox_id: &str, generation: &str) -> Self {
|
||||
let mut names = Self::new(sandbox_id);
|
||||
let generation = generation
|
||||
.chars()
|
||||
.filter(char::is_ascii_alphanumeric)
|
||||
.take(12)
|
||||
.collect::<String>()
|
||||
.to_ascii_lowercase();
|
||||
names.sandbox_secret = format!("{}-{generation}", names.sandbox_secret);
|
||||
names.supervisor_secret = format!("{}-{generation}", names.supervisor_secret);
|
||||
names
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn pair_label_value(sandbox_id: &str) -> String {
|
||||
sandbox_id.to_ascii_lowercase()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn workload_fence(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
sandbox_id: &str,
|
||||
boundary_port: u16,
|
||||
) -> KubernetesProxyPodNetworkFence {
|
||||
KubernetesProxyPodNetworkFenceSpec {
|
||||
namespace: namespace.to_string(),
|
||||
policy_name: names.workload_policy.clone(),
|
||||
pair_label_value: pair_label_value(sandbox_id),
|
||||
boundary_port,
|
||||
}
|
||||
.provision()
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn boundary_service(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
sandbox_id: &str,
|
||||
boundary_port: u16,
|
||||
owner: OwnerReference,
|
||||
) -> Service {
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"apiVersion": "v1",
|
||||
"kind": "Service",
|
||||
"metadata": {
|
||||
"name": names.boundary_service,
|
||||
"namespace": namespace,
|
||||
"ownerReferences": [owner],
|
||||
"labels": common_labels(sandbox_id, "boundary-service"),
|
||||
},
|
||||
"spec": {
|
||||
"selector": pair_labels(sandbox_id, "workload"),
|
||||
"ports": [{"name": "boundary", "protocol": "TCP", "port": boundary_port, "targetPort": boundary_port}],
|
||||
}
|
||||
}))
|
||||
.expect("boundary Service renderer must produce a valid object")
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments, clippy::similar_names)]
|
||||
#[must_use]
|
||||
pub fn control_deployment(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
sandbox_id: &str,
|
||||
sandbox_name: &str,
|
||||
gateway_id: &str,
|
||||
supervisor_image: &str,
|
||||
supervisor_pull_policy: &str,
|
||||
service_account_name: &str,
|
||||
control_uid: u32,
|
||||
control_gid: u32,
|
||||
image_pull_secrets: &[String],
|
||||
grpc_endpoint: &str,
|
||||
client_tls_secret_name: &str,
|
||||
main_process_spec: &str,
|
||||
log_level: &str,
|
||||
sa_token_ttl_secs: i64,
|
||||
https_proxy: Option<&str>,
|
||||
no_proxy: Option<&str>,
|
||||
proxy_auth_secret: Option<(&str, &str)>,
|
||||
proxy_auth_allow_insecure: bool,
|
||||
proxy_connect_by_hostname: bool,
|
||||
provider_spiffe_socket_path: Option<&str>,
|
||||
owner: OwnerReference,
|
||||
) -> Deployment {
|
||||
let labels = control_labels(sandbox_id, gateway_id);
|
||||
let mut environment = vec![
|
||||
env_var(
|
||||
"OPENSHELL_ADMITTED_ISOLATION_BACKEND",
|
||||
crate::isolation::BACKEND_NAME,
|
||||
),
|
||||
env_var("OPENSHELL_ENDPOINT", grpc_endpoint),
|
||||
env_var("OPENSHELL_SANDBOX_ID", sandbox_id),
|
||||
env_var("OPENSHELL_SANDBOX", sandbox_name),
|
||||
env_var("OPENSHELL_MAIN_PROCESS_SPEC", main_process_spec),
|
||||
env_var(
|
||||
"OPENSHELL_K8S_SA_TOKEN_FILE",
|
||||
"/var/run/secrets/openshell/token",
|
||||
),
|
||||
env_var("OPENSHELL_SSH_SOCKET_PATH", "/run/openshell/ssh.sock"),
|
||||
env_var(openshell_core::sandbox_env::SSH_SOCKET_SHARED, "true"),
|
||||
env_var("OPENSHELL_PROXY_TLS_DIR", "/run/openshell/proxy-tls"),
|
||||
env_var(
|
||||
openshell_core::sandbox_env::PROXY_CA_CERT,
|
||||
PROXY_CA_CERTIFICATE_PATH,
|
||||
),
|
||||
env_var(
|
||||
openshell_core::sandbox_env::PROXY_CA_KEY,
|
||||
PROXY_CA_PRIVATE_KEY_PATH,
|
||||
),
|
||||
env_var("OPENSHELL_LOG_LEVEL", log_level),
|
||||
env_var(
|
||||
openshell_core::sandbox_env::TELEMETRY_ENABLED,
|
||||
openshell_core::telemetry::enabled_env_value(),
|
||||
),
|
||||
env_var(
|
||||
openshell_core::sandbox_env::NETWORK_RUNTIME_CAPABILITIES,
|
||||
"",
|
||||
),
|
||||
];
|
||||
let mut volume_mounts = vec![
|
||||
serde_json::json!({"name": "bootstrap", "mountPath": "/.openshell/supervisor", "readOnly": true}),
|
||||
serde_json::json!({"name": "sa-token", "mountPath": "/var/run/secrets/openshell", "readOnly": true}),
|
||||
serde_json::json!({"name": "run", "mountPath": "/run/openshell"}),
|
||||
serde_json::json!({"name": "logs", "mountPath": "/var/log"}),
|
||||
];
|
||||
let mut volumes = vec![
|
||||
serde_json::json!({"name": "bootstrap", "secret": {"secretName": names.supervisor_secret, "defaultMode": 0o440}}),
|
||||
serde_json::json!({"name": "sa-token", "projected": {"sources": [{"serviceAccountToken": {"audience": "openshell-gateway", "expirationSeconds": sa_token_ttl_secs, "path": "token"}}], "defaultMode": 0o440}}),
|
||||
serde_json::json!({"name": "run", "emptyDir": {}}),
|
||||
serde_json::json!({"name": "logs", "emptyDir": {}}),
|
||||
];
|
||||
if !client_tls_secret_name.is_empty() {
|
||||
environment.extend([
|
||||
env_var("OPENSHELL_TLS_CA", "/var/run/secrets/openshell-tls/ca.crt"),
|
||||
env_var(
|
||||
"OPENSHELL_TLS_CERT",
|
||||
"/var/run/secrets/openshell-tls/tls.crt",
|
||||
),
|
||||
env_var(
|
||||
"OPENSHELL_TLS_KEY",
|
||||
"/var/run/secrets/openshell-tls/tls.key",
|
||||
),
|
||||
]);
|
||||
volume_mounts.push(serde_json::json!({"name": "client-tls", "mountPath": "/var/run/secrets/openshell-tls", "readOnly": true}));
|
||||
volumes.push(serde_json::json!({"name": "client-tls", "secret": {"secretName": client_tls_secret_name, "defaultMode": 0o440}}));
|
||||
}
|
||||
let mut command = vec![
|
||||
"/openshell-supervisor".to_string(),
|
||||
"--topology-backend-name".to_string(),
|
||||
crate::isolation::BACKEND_NAME.to_string(),
|
||||
"--topology-payload-file".to_string(),
|
||||
TOPOLOGY_PAYLOAD_PATH.to_string(),
|
||||
"--auth-bundle-file".to_string(),
|
||||
SUPERVISOR_AUTH_BUNDLE_PATH.to_string(),
|
||||
"--workdir".to_string(),
|
||||
"/sandbox".to_string(),
|
||||
"--health-socket-path".to_string(),
|
||||
CONTROL_HEALTH_SOCKET_PATH.to_string(),
|
||||
];
|
||||
if let Some(url) = https_proxy {
|
||||
command.extend(["--upstream-proxy".to_string(), url.to_string()]);
|
||||
}
|
||||
if let Some(hosts) = no_proxy {
|
||||
command.extend(["--upstream-no-proxy".to_string(), hosts.to_string()]);
|
||||
}
|
||||
if proxy_auth_secret.is_some() {
|
||||
command.extend([
|
||||
"--upstream-proxy-auth-file".to_string(),
|
||||
openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH.to_string(),
|
||||
]);
|
||||
}
|
||||
if proxy_auth_allow_insecure {
|
||||
command.push("--upstream-proxy-auth-allow-insecure".to_string());
|
||||
}
|
||||
if proxy_connect_by_hostname {
|
||||
command.push("--upstream-proxy-connect-by-hostname".to_string());
|
||||
}
|
||||
if let Some((secret_name, secret_key)) = proxy_auth_secret {
|
||||
let auth_path = Path::new(openshell_core::container_paths::UPSTREAM_PROXY_AUTH_MOUNT_PATH);
|
||||
volume_mounts.push(serde_json::json!({
|
||||
"name": "upstream-proxy-auth",
|
||||
"mountPath": auth_path.parent().and_then(Path::to_str).expect("auth path has parent"),
|
||||
"readOnly": true
|
||||
}));
|
||||
volumes.push(serde_json::json!({
|
||||
"name": "upstream-proxy-auth",
|
||||
"secret": {"secretName": secret_name, "defaultMode": 0o440, "items": [{
|
||||
"key": secret_key,
|
||||
"path": auth_path.file_name().and_then(|name| name.to_str()).expect("auth path has file name")
|
||||
}]}
|
||||
}));
|
||||
}
|
||||
if let Some(socket_path) = provider_spiffe_socket_path {
|
||||
environment.push(env_var(
|
||||
openshell_core::sandbox_env::PROVIDER_SPIFFE_WORKLOAD_API_SOCKET,
|
||||
socket_path,
|
||||
));
|
||||
volume_mounts.push(serde_json::json!({
|
||||
"name": "spiffe-workload-api",
|
||||
"mountPath": Path::new(socket_path).parent().and_then(Path::to_str).expect("SPIFFE socket has parent"),
|
||||
"readOnly": true
|
||||
}));
|
||||
volumes.push(serde_json::json!({
|
||||
"name": "spiffe-workload-api",
|
||||
"csi": {"driver": "csi.spiffe.io", "readOnly": true}
|
||||
}));
|
||||
}
|
||||
let mut container = serde_json::json!({
|
||||
"name": "supervisor",
|
||||
"image": supervisor_image,
|
||||
"command": command,
|
||||
"terminationMessagePolicy": "FallbackToLogsOnError",
|
||||
"env": environment,
|
||||
"readinessProbe": {
|
||||
"exec": {"command": [
|
||||
"/openshell-supervisor",
|
||||
"health",
|
||||
"--socket",
|
||||
CONTROL_HEALTH_SOCKET_PATH
|
||||
]},
|
||||
"periodSeconds": 1,
|
||||
"failureThreshold": 3
|
||||
},
|
||||
"securityContext": {
|
||||
"runAsUser": control_uid,
|
||||
"runAsGroup": control_gid,
|
||||
"runAsNonRoot": true,
|
||||
"readOnlyRootFilesystem": true,
|
||||
"allowPrivilegeEscalation": false,
|
||||
"capabilities": {"drop": ["ALL"]}
|
||||
},
|
||||
"volumeMounts": volume_mounts,
|
||||
});
|
||||
if !supervisor_pull_policy.is_empty() {
|
||||
container["imagePullPolicy"] = serde_json::json!(supervisor_pull_policy);
|
||||
}
|
||||
serde_json::from_value(serde_json::json!({
|
||||
"apiVersion": "apps/v1",
|
||||
"kind": "Deployment",
|
||||
"metadata": {"name": names.control_deployment, "namespace": namespace, "ownerReferences": [owner], "labels": labels},
|
||||
"spec": {
|
||||
"replicas": 0,
|
||||
"strategy": {"type": "Recreate"},
|
||||
"selector": {"matchLabels": pair_labels(sandbox_id, "supervisor")},
|
||||
"template": {
|
||||
"metadata": {"labels": control_labels(sandbox_id, gateway_id), "annotations": {"openshell.ai/sandbox-id": sandbox_id}},
|
||||
"spec": {
|
||||
"serviceAccountName": service_account_name,
|
||||
"imagePullSecrets": image_pull_secrets.iter().map(|name| serde_json::json!({"name": name})).collect::<Vec<_>>(),
|
||||
"automountServiceAccountToken": false,
|
||||
"securityContext": {
|
||||
"fsGroup": control_gid,
|
||||
"fsGroupChangePolicy": "OnRootMismatch",
|
||||
"seccompProfile": {"type": "RuntimeDefault"}
|
||||
},
|
||||
"restartPolicy": "Always",
|
||||
"containers": [container],
|
||||
"volumes": volumes
|
||||
}
|
||||
}
|
||||
}
|
||||
}))
|
||||
.expect("control Deployment renderer must produce a valid object")
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn control_egress_policy(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
sandbox_id: &str,
|
||||
owner: OwnerReference,
|
||||
) -> NetworkPolicy {
|
||||
NetworkPolicy {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(names.control_policy.clone()),
|
||||
namespace: Some(namespace.to_string()),
|
||||
owner_references: Some(vec![owner]),
|
||||
labels: Some(common_labels(sandbox_id, "control-egress")),
|
||||
..Default::default()
|
||||
},
|
||||
spec: Some(NetworkPolicySpec {
|
||||
pod_selector: LabelSelector {
|
||||
match_labels: Some(pair_labels(sandbox_id, "supervisor")),
|
||||
..Default::default()
|
||||
},
|
||||
policy_types: Some(vec!["Egress".to_string()]),
|
||||
// Control is the policy-enforcing egress principal. Namespace-wide
|
||||
// default-deny policies must not prevent its gateway/DNS/upstream dials.
|
||||
egress: Some(vec![NetworkPolicyEgressRule::default()]),
|
||||
..Default::default()
|
||||
}),
|
||||
status: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
#[must_use]
|
||||
pub fn sandbox_bootstrap_secret(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
sandbox_id: &str,
|
||||
boundary_config: Vec<u8>,
|
||||
boundary_certificate: Vec<u8>,
|
||||
boundary_private_key: Vec<u8>,
|
||||
owner: OwnerReference,
|
||||
) -> Secret {
|
||||
Secret {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(names.sandbox_secret.clone()),
|
||||
namespace: Some(namespace.to_string()),
|
||||
owner_references: Some(vec![owner]),
|
||||
labels: Some(common_labels(sandbox_id, SANDBOX_SECRET_COMPONENT)),
|
||||
..Default::default()
|
||||
},
|
||||
data: Some(BTreeMap::from([
|
||||
(BOUNDARY_CONFIG_KEY.to_string(), ByteString(boundary_config)),
|
||||
(
|
||||
BOUNDARY_CERTIFICATE_KEY.to_string(),
|
||||
ByteString(boundary_certificate),
|
||||
),
|
||||
(
|
||||
BOUNDARY_PRIVATE_KEY.to_string(),
|
||||
ByteString(boundary_private_key),
|
||||
),
|
||||
])),
|
||||
immutable: Some(true),
|
||||
type_: Some("Opaque".to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn supervisor_bootstrap_secret(
|
||||
namespace: &str,
|
||||
names: &ProxyPodNames,
|
||||
sandbox_id: &str,
|
||||
topology_payload: Vec<u8>,
|
||||
supervisor_auth_bundle: Vec<u8>,
|
||||
proxy_ca_certificate: Vec<u8>,
|
||||
proxy_ca_private_key: Vec<u8>,
|
||||
owner: OwnerReference,
|
||||
) -> Secret {
|
||||
Secret {
|
||||
metadata: ObjectMeta {
|
||||
name: Some(names.supervisor_secret.clone()),
|
||||
namespace: Some(namespace.to_string()),
|
||||
owner_references: Some(vec![owner]),
|
||||
labels: Some(common_labels(sandbox_id, SUPERVISOR_SECRET_COMPONENT)),
|
||||
..Default::default()
|
||||
},
|
||||
data: Some(BTreeMap::from([
|
||||
(
|
||||
TOPOLOGY_PAYLOAD_KEY.to_string(),
|
||||
ByteString(topology_payload),
|
||||
),
|
||||
(
|
||||
SUPERVISOR_AUTH_BUNDLE_KEY.to_string(),
|
||||
ByteString(supervisor_auth_bundle),
|
||||
),
|
||||
(
|
||||
PROXY_CA_CERTIFICATE_KEY.to_string(),
|
||||
ByteString(proxy_ca_certificate),
|
||||
),
|
||||
(
|
||||
PROXY_CA_PRIVATE_KEY.to_string(),
|
||||
ByteString(proxy_ca_private_key),
|
||||
),
|
||||
])),
|
||||
immutable: Some(true),
|
||||
type_: Some("Opaque".to_string()),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn sandbox_owner_reference(
|
||||
name: &str,
|
||||
uid: &str,
|
||||
api_version: &str,
|
||||
controller: bool,
|
||||
) -> OwnerReference {
|
||||
OwnerReference {
|
||||
api_version: api_version.to_string(),
|
||||
kind: "Sandbox".to_string(),
|
||||
name: name.to_string(),
|
||||
uid: uid.to_string(),
|
||||
controller: controller.then_some(true),
|
||||
// The driver's RBAC intentionally does not permit mutating Sandbox
|
||||
// finalizers. Kubernetes garbage collection does not require this bit,
|
||||
// and setting it would make admission fail under
|
||||
// OwnerReferencesPermissionEnforcement.
|
||||
block_owner_deletion: Some(false),
|
||||
}
|
||||
}
|
||||
|
||||
fn pair_labels(sandbox_id: &str, role: &str) -> BTreeMap<String, String> {
|
||||
BTreeMap::from([
|
||||
(
|
||||
BOUNDARY_PAIR_LABEL.to_string(),
|
||||
pair_label_value(sandbox_id),
|
||||
),
|
||||
(BOUNDARY_ROLE_LABEL.to_string(), role.to_string()),
|
||||
])
|
||||
}
|
||||
|
||||
fn common_labels(sandbox_id: &str, component: &str) -> BTreeMap<String, String> {
|
||||
BTreeMap::from([
|
||||
(
|
||||
"openshell.ai/managed-by".to_string(),
|
||||
"openshell".to_string(),
|
||||
),
|
||||
(
|
||||
"openshell.ai/sandbox-id".to_string(),
|
||||
sandbox_id.to_string(),
|
||||
),
|
||||
("openshell.ai/component".to_string(), component.to_string()),
|
||||
])
|
||||
}
|
||||
|
||||
fn control_labels(sandbox_id: &str, gateway_id: &str) -> BTreeMap<String, String> {
|
||||
let mut labels = common_labels(sandbox_id, "supervisor");
|
||||
labels.extend(pair_labels(sandbox_id, "supervisor"));
|
||||
labels.insert(
|
||||
"openshell.ai/gateway-id".to_string(),
|
||||
gateway_id.to_string(),
|
||||
);
|
||||
labels
|
||||
}
|
||||
|
||||
fn env_var(name: &str, value: &str) -> serde_json::Value {
|
||||
serde_json::json!({"name": name, "value": value})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn owner() -> OwnerReference {
|
||||
sandbox_owner_reference("demo", "uid-1", "agents.x-k8s.io/v1beta1", true)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn service_selects_only_the_workload_boundary() {
|
||||
let names = ProxyPodNames::new("4b67c0d0-1111-2222-3333-444444444444");
|
||||
let service = boundary_service("sandbox", &names, "pair", 5500, owner());
|
||||
assert_eq!(
|
||||
service.spec.unwrap().selector.unwrap()[BOUNDARY_ROLE_LABEL],
|
||||
"workload"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn owner_reference_does_not_require_finalizer_mutation_permission() {
|
||||
assert_eq!(owner().block_owner_deletion, Some(false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_policy_explicitly_allows_control_egress() {
|
||||
let names = ProxyPodNames::new("pair");
|
||||
let policy = control_egress_policy("sandbox", &names, "pair", owner());
|
||||
let spec = policy.spec.unwrap();
|
||||
assert_eq!(spec.policy_types.unwrap(), ["Egress"]);
|
||||
assert_eq!(spec.egress.unwrap().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_deployment_is_singleton_ready_and_unprivileged() {
|
||||
let names = ProxyPodNames::new("pair");
|
||||
let deployment = control_deployment(
|
||||
"sandbox",
|
||||
&names,
|
||||
"pair",
|
||||
"demo",
|
||||
"gateway",
|
||||
"supervisor:latest",
|
||||
"IfNotPresent",
|
||||
"sandbox-sa",
|
||||
1000,
|
||||
1000,
|
||||
&["registry-credentials".to_string()],
|
||||
"https://gateway:8080",
|
||||
"client-tls",
|
||||
"{}",
|
||||
"info",
|
||||
600,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
false,
|
||||
false,
|
||||
None,
|
||||
owner(),
|
||||
);
|
||||
let spec = deployment.spec.as_ref().unwrap();
|
||||
assert_eq!(
|
||||
spec.strategy
|
||||
.as_ref()
|
||||
.and_then(|strategy| strategy.type_.as_deref()),
|
||||
Some("Recreate")
|
||||
);
|
||||
let container = &spec.template.spec.as_ref().unwrap().containers[0];
|
||||
let pod_spec = spec.template.spec.as_ref().unwrap();
|
||||
assert_eq!(pod_spec.automount_service_account_token, Some(false));
|
||||
assert_eq!(
|
||||
pod_spec.image_pull_secrets.as_ref().unwrap()[0]
|
||||
.name
|
||||
.as_deref(),
|
||||
Some("registry-credentials")
|
||||
);
|
||||
let pod_security =
|
||||
serde_json::to_value(pod_spec.security_context.as_ref().unwrap()).unwrap();
|
||||
assert_eq!(pod_security["fsGroup"], 1000);
|
||||
assert_eq!(pod_security["seccompProfile"]["type"], "RuntimeDefault");
|
||||
let container_security =
|
||||
serde_json::to_value(container.security_context.as_ref().unwrap()).unwrap();
|
||||
assert_eq!(container_security["runAsUser"], 1000);
|
||||
assert_eq!(container_security["runAsNonRoot"], true);
|
||||
assert_eq!(container_security["readOnlyRootFilesystem"], true);
|
||||
assert_eq!(
|
||||
container_security["capabilities"]["drop"],
|
||||
serde_json::json!(["ALL"])
|
||||
);
|
||||
assert_eq!(
|
||||
container
|
||||
.readiness_probe
|
||||
.as_ref()
|
||||
.and_then(|probe| probe.exec.as_ref())
|
||||
.and_then(|exec| exec.command.as_ref()),
|
||||
Some(&vec![
|
||||
"/openshell-supervisor".to_string(),
|
||||
"health".to_string(),
|
||||
"--socket".to_string(),
|
||||
CONTROL_HEALTH_SOCKET_PATH.to_string(),
|
||||
])
|
||||
);
|
||||
let command = container.command.as_ref().unwrap();
|
||||
assert!(
|
||||
command
|
||||
.windows(2)
|
||||
.any(|args| args == ["--health-socket-path", CONTROL_HEALTH_SOCKET_PATH])
|
||||
);
|
||||
let env = container.env.as_ref().unwrap();
|
||||
let env_value = |name: &str| {
|
||||
env.iter()
|
||||
.find(|variable| variable.name == name)
|
||||
.and_then(|variable| variable.value.as_deref())
|
||||
};
|
||||
assert_eq!(
|
||||
env_value(openshell_core::sandbox_env::SSH_SOCKET_SHARED),
|
||||
Some("true")
|
||||
);
|
||||
assert_eq!(
|
||||
env_value(openshell_core::sandbox_env::PROXY_CA_CERT),
|
||||
Some(PROXY_CA_CERTIFICATE_PATH)
|
||||
);
|
||||
assert_eq!(
|
||||
env_value(openshell_core::sandbox_env::PROXY_CA_KEY),
|
||||
Some(PROXY_CA_PRIVATE_KEY_PATH)
|
||||
);
|
||||
let mount = container
|
||||
.volume_mounts
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.find(|mount| mount.name == "bootstrap")
|
||||
.expect("durable supervisor material is mounted into supervisor");
|
||||
assert_eq!(mount.mount_path, "/.openshell/supervisor");
|
||||
assert_eq!(mount.read_only, Some(true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bootstrap_secrets_are_immutable_and_split_by_trust_domain() {
|
||||
let names = ProxyPodNames::new("pair");
|
||||
let sandbox = sandbox_bootstrap_secret(
|
||||
"sandbox",
|
||||
&names,
|
||||
"pair",
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
owner(),
|
||||
);
|
||||
assert_eq!(
|
||||
sandbox.metadata.labels.as_ref().unwrap()["openshell.ai/component"],
|
||||
SANDBOX_SECRET_COMPONENT
|
||||
);
|
||||
assert_eq!(sandbox.immutable, Some(true));
|
||||
let sandbox_keys = sandbox
|
||||
.data
|
||||
.unwrap()
|
||||
.into_keys()
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
sandbox_keys,
|
||||
std::collections::BTreeSet::from([
|
||||
BOUNDARY_CERTIFICATE_KEY.to_string(),
|
||||
BOUNDARY_CONFIG_KEY.to_string(),
|
||||
BOUNDARY_PRIVATE_KEY.to_string(),
|
||||
])
|
||||
);
|
||||
|
||||
let supervisor = supervisor_bootstrap_secret(
|
||||
"sandbox",
|
||||
&names,
|
||||
"pair",
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
Vec::new(),
|
||||
owner(),
|
||||
);
|
||||
assert_eq!(
|
||||
supervisor.metadata.labels.as_ref().unwrap()["openshell.ai/component"],
|
||||
SUPERVISOR_SECRET_COMPONENT
|
||||
);
|
||||
assert_eq!(supervisor.immutable, Some(true));
|
||||
let supervisor_keys = supervisor
|
||||
.data
|
||||
.unwrap()
|
||||
.into_keys()
|
||||
.collect::<std::collections::BTreeSet<_>>();
|
||||
assert_eq!(
|
||||
supervisor_keys,
|
||||
std::collections::BTreeSet::from([
|
||||
PROXY_CA_CERTIFICATE_KEY.to_string(),
|
||||
PROXY_CA_PRIVATE_KEY.to_string(),
|
||||
SUPERVISOR_AUTH_BUNDLE_KEY.to_string(),
|
||||
TOPOLOGY_PAYLOAD_KEY.to_string(),
|
||||
])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_proxy_ca_material_is_pem_encoded() {
|
||||
let material = generate_proxy_ca_material().unwrap();
|
||||
assert!(
|
||||
material
|
||||
.certificate_pem
|
||||
.starts_with("-----BEGIN CERTIFICATE-----")
|
||||
);
|
||||
assert!(material.private_key_pem.contains("PRIVATE KEY"));
|
||||
}
|
||||
}
|
||||
@@ -43,12 +43,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version <vers
|
||||
See the full [OpenShift install guide](https://docs.nvidia.com/openshell/latest/kubernetes/openshift) for details. Quick start:
|
||||
|
||||
```shell
|
||||
# Precreate the openshell namespace so we can create the SCC cluster role
|
||||
# Precreate the openshell namespace
|
||||
oc create ns openshell
|
||||
|
||||
# Sandboxes are deployed into the openshell namespace and use the openshell-sandbox service account
|
||||
oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell
|
||||
|
||||
# Deploy openshell with overrides to allow SCC assignment of fsGroup and runAsUser for the gateway
|
||||
helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version <version> -n openshell \
|
||||
--set server.disableTls=true \
|
||||
@@ -230,7 +227,6 @@ discovery endpoint or its TLS CA.
|
||||
| securityContext.capabilities.drop | list | `["ALL"]` | Linux capabilities dropped from the gateway container. |
|
||||
| securityContext.runAsNonRoot | bool | `true` | Require the gateway container to run as a non-root user. |
|
||||
| securityContext.runAsUser | int | `1000` | UID assigned to the gateway container. |
|
||||
| server.appArmorProfile | string | `"Unconfined"` | Kubernetes AppArmor profile requested for sandbox agent containers. Default Unconfined avoids runtime/default AppArmor blocking the supervisor's network namespace mount setup on AppArmor-enabled nodes. Set to "" to omit the field, "RuntimeDefault" to force the runtime default profile, or "Localhost/profile-name" for an operator-managed localhost profile. |
|
||||
| server.auth.allowUnauthenticatedUsers | bool | `false` | UNSAFE: accept unauthenticated CLI/user requests as a local developer principal. Intended only for trusted local Skaffold/k3d development or a fully trusted fronting proxy. Leave false for shared or production clusters. |
|
||||
| server.credentialDrivers.kubernetesSecrets.allowReferenceNamespace | bool | `false` | Deprecated compatibility field. Credential storage no longer supports user-authored namespace references. |
|
||||
| server.credentialDrivers.kubernetesSecrets.enabled | bool | `false` | Enable the in-tree Kubernetes Secret credential driver. WARNING: The RBAC Role grants read/write access to ALL Secrets in the configured namespace. Use a dedicated namespace to limit blast radius. |
|
||||
@@ -300,10 +296,8 @@ discovery endpoint or its TLS CA.
|
||||
| supervisor.image.pullPolicy | string | `nil` | Sandbox supervisor pull policy. Leave unset to use the Kubernetes image default. Prefer always, if_not_present, or never; the chart also accepts legacy Kubernetes spellings Always, IfNotPresent, and Never. |
|
||||
| supervisor.image.repository | string | `"ghcr.io/nvidia/openshell/supervisor"` | Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set. |
|
||||
| supervisor.image.tag | string | `""` | Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed. |
|
||||
| supervisor.sidecar.processBinaryAwareNetworkPolicy | bool | `true` | Keep process/binary-aware network policy enabled in sidecar topology. When false, the network sidecar runs as proxyUid, drops the extra /proc inspection capabilities, and enforces endpoint/L7 policy without matching policy.binaries. |
|
||||
| supervisor.sidecar.proxyUid | int | `1337` | UID for relaxed long-running network sidecars in sidecar topology. Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants the required /proc inspection capabilities into the effective set. The network init container installs nftables rules that exempt the effective sidecar UID. |
|
||||
| supervisor.sideloadMethod | string | `""` | How the supervisor binary is delivered into sandbox pods. Empty (default) = auto-detect from cluster version: K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36) K8s < v1.35 -> "init-container" (copies via init container + emptyDir) On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled, set this to "image-volume" explicitly. |
|
||||
| supervisor.topology | string | `"combined"` | Supervisor pod topology for Kubernetes sandboxes. "combined" runs the current single supervisor container in the agent pod. "sidecar" runs network enforcement in a dedicated sidecar and the process supervisor as a low-capability wrapper in the agent container. |
|
||||
| supervisor.proxyPod.boundaryPort | int | `5500` | Workload boundary TLS listener port. |
|
||||
| supervisor.proxyPod.networkPolicyEnforced | bool | `false` | Required operator acknowledgement that the cluster CNI enforces NetworkPolicy. |
|
||||
| tolerations | list | `[]` | Tolerations for the gateway pod. |
|
||||
| upstreamProxy | object | `{"authAllowInsecure":false,"authSecret":{"key":"","name":""},"connectByHostname":false,"noProxy":"","url":""}` | Operator-owned corporate forward proxy for policy-approved TLS egress from Kubernetes sandboxes. The workload cannot select or override it. |
|
||||
| upstreamProxy.authAllowInsecure | bool | `false` | Required when authSecret is configured because Basic auth to an HTTP proxy is cleartext. |
|
||||
|
||||
@@ -43,12 +43,9 @@ helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version <vers
|
||||
See the full [OpenShift install guide](https://docs.nvidia.com/openshell/latest/kubernetes/openshift) for details. Quick start:
|
||||
|
||||
```shell
|
||||
# Precreate the openshell namespace so we can create the SCC cluster role
|
||||
# Precreate the openshell namespace
|
||||
oc create ns openshell
|
||||
|
||||
# Sandboxes are deployed into the openshell namespace and use the openshell-sandbox service account
|
||||
oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell
|
||||
|
||||
# Deploy openshell with overrides to allow SCC assignment of fsGroup and runAsUser for the gateway
|
||||
helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart --version <version> -n openshell \
|
||||
--set server.disableTls=true \
|
||||
|
||||
@@ -3,9 +3,6 @@
|
||||
|
||||
# The Kubernetes corporate-proxy e2e wrapper supplies the generated proxy URL
|
||||
# and creates `openshell-e2e-proxy-auth` before Helm installs the gateway.
|
||||
supervisor:
|
||||
topology: sidecar
|
||||
|
||||
upstreamProxy:
|
||||
authSecret:
|
||||
name: openshell-e2e-proxy-auth
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
supervisor:
|
||||
proxyPod:
|
||||
networkPolicyEnforced: true
|
||||
@@ -1,24 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology under
|
||||
# a Kata RuntimeClass.
|
||||
#
|
||||
# Use with e2e/with-kube-gateway.sh by setting:
|
||||
# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar-kata.yaml
|
||||
# The e2e wrapper supplies the image repository and tag through OPENSHELL_REGISTRY
|
||||
# and IMAGE_TAG for existing-cluster runs.
|
||||
|
||||
supervisor:
|
||||
# Use the sidecar topology under Kata so network enforcement runs in the
|
||||
# sidecar and the sandbox agent container stays low-privilege.
|
||||
topology: sidecar
|
||||
sidecar:
|
||||
# Keep strict process/binary-aware network policy enabled for the Kata
|
||||
# validation path. Set this false only when intentionally validating the
|
||||
# documented endpoint/L7-only downgrade mode.
|
||||
processBinaryAwareNetworkPolicy: true
|
||||
|
||||
# Kata validation clusters normally install this RuntimeClass.
|
||||
server:
|
||||
defaultRuntimeClassName: kata-qemu
|
||||
@@ -1,18 +0,0 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# CI/dev overlay for exercising the Kubernetes supervisor sidecar topology.
|
||||
#
|
||||
# Merge after values.yaml and ci/values-skaffold.yaml:
|
||||
# helm install ... -f values.yaml -f ci/values-skaffold.yaml -f ci/values-sidecar.yaml
|
||||
#
|
||||
# Or set:
|
||||
# OPENSHELL_E2E_KUBE_EXTRA_VALUES=deploy/helm/openshell/ci/values-sidecar.yaml
|
||||
# before running `mise run e2e:kubernetes`.
|
||||
supervisor:
|
||||
topology: sidecar
|
||||
sidecar:
|
||||
# The strict sidecar default requires cross-container /proc identity access.
|
||||
# CI/dev e2e uses the explicit downgraded mode so endpoint and L7 policy
|
||||
# coverage remains runnable on local k3d while that path is hardened.
|
||||
processBinaryAwareNetworkPolicy: false
|
||||
@@ -136,8 +136,6 @@ deploy:
|
||||
# To enable SPIFFE/SPIRE provider token grants (requires the
|
||||
# spire-crds and spire releases above):
|
||||
#- ci/values-spire.yaml
|
||||
# To exercise the Kubernetes supervisor sidecar topology:
|
||||
#- ci/values-sidecar.yaml
|
||||
# To test multi-replica external PostgreSQL behavior:
|
||||
#- ci/values-high-availability.yaml
|
||||
setValueTemplates:
|
||||
@@ -146,20 +144,6 @@ deploy:
|
||||
supervisor.image.repository: '{{.IMAGE_REPO_openshell_supervisor}}'
|
||||
supervisor.image.tag: '{{.IMAGE_TAG_openshell_supervisor}}'
|
||||
profiles:
|
||||
- name: sidecar
|
||||
patches:
|
||||
- op: add
|
||||
path: /deploy/helm/releases/0/valuesFiles/-
|
||||
value: ci/values-sidecar.yaml
|
||||
- name: sidecar-mtls
|
||||
patches:
|
||||
- op: add
|
||||
path: /deploy/helm/releases/0/valuesFiles/-
|
||||
value: ci/values-sidecar.yaml
|
||||
- op: add
|
||||
path: /deploy/helm/releases/0/setValues
|
||||
value:
|
||||
server.disableTls: "false"
|
||||
- name: credential-driver-kubernetes-secrets
|
||||
patches:
|
||||
- op: add
|
||||
|
||||
@@ -194,25 +194,6 @@ the in-cluster Service DNS, release namespace, service port, and disableTls
|
||||
flag — so the default value works for any release name or namespace without
|
||||
override.
|
||||
*/}}
|
||||
{{/*
|
||||
Supervisor sideload method. When supervisor.sideloadMethod is set, use it
|
||||
verbatim. Otherwise auto-detect from the cluster version: the ImageVolume
|
||||
feature gate is enabled by default starting in K8s v1.35 (GA in v1.36).
|
||||
Clusters on v1.33-v1.34 can opt in by setting sideloadMethod explicitly
|
||||
after enabling the feature gate.
|
||||
*/}}
|
||||
{{- define "openshell.supervisorSideloadMethod" -}}
|
||||
{{- if .Values.supervisor.sideloadMethod -}}
|
||||
{{- .Values.supervisor.sideloadMethod -}}
|
||||
{{- else -}}
|
||||
{{- if semverCompare ">=1.35-0" .Capabilities.KubeVersion.Version -}}
|
||||
image-volume
|
||||
{{- else -}}
|
||||
init-container
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- end }}
|
||||
|
||||
{{- define "openshell.grpcEndpoint" -}}
|
||||
{{- if .Values.server.grpcEndpoint -}}
|
||||
{{- .Values.server.grpcEndpoint -}}
|
||||
|
||||
@@ -72,6 +72,7 @@ rules:
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- patch
|
||||
{{- end }}
|
||||
{{- $copiedSecretNames := list }}
|
||||
{{- if and (ne $workspaceMode "shared") (not .Values.server.disableTls) }}
|
||||
@@ -84,6 +85,23 @@ rules:
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if ne $workspaceMode "shared" }}
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments"]
|
||||
verbs: ["create", "delete", "get", "list", "patch", "watch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["replicasets"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
verbs: ["create", "delete", "get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["networkpolicies"]
|
||||
verbs: ["create", "delete", "get", "list"]
|
||||
{{- end }}
|
||||
{{- $copiedSecretNames = uniq $copiedSecretNames }}
|
||||
{{- if $copiedSecretNames }}
|
||||
# Copy only explicitly configured TLS and image-pull Secrets into workspace
|
||||
|
||||
@@ -160,8 +160,6 @@ data:
|
||||
{{- if .Values.server.drivers.kubernetes.operatorNamespaceFile }}
|
||||
operator_namespace_file = {{ .Values.server.drivers.kubernetes.operatorNamespaceFile | quote }}
|
||||
{{- end }}
|
||||
supervisor_sideload_method = {{ include "openshell.supervisorSideloadMethod" . | quote }}
|
||||
topology = {{ .Values.supervisor.topology | default "combined" | quote }}
|
||||
sa_token_ttl_secs = {{ .Values.server.sandboxJwt.k8sSaTokenTtlSecs | default 3600 }}
|
||||
{{- if .Values.upstreamProxy.url }}
|
||||
https_proxy = {{ .Values.upstreamProxy.url | quote }}
|
||||
@@ -205,9 +203,6 @@ data:
|
||||
{{- if .Values.server.defaultRuntimeClassName }}
|
||||
default_runtime_class_name = {{ .Values.server.defaultRuntimeClassName | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.server.appArmorProfile }}
|
||||
app_armor_profile = {{ .Values.server.appArmorProfile | quote }}
|
||||
{{- end }}
|
||||
{{- if .Values.supervisor.image.pullPolicy }}
|
||||
supervisor_image_pull_policy = {{ include "openshell.canonicalImagePullPolicy" .Values.supervisor.image.pullPolicy | quote }}
|
||||
{{- end }}
|
||||
@@ -217,9 +212,9 @@ data:
|
||||
gateway_namespace = {{ .Release.Namespace | quote }}
|
||||
gateway_pod_selector = { "app.kubernetes.io/name" = {{ include "openshell.name" . | quote }}, "app.kubernetes.io/instance" = {{ .Release.Name | quote }} }
|
||||
|
||||
[openshell.drivers.kubernetes.sidecar]
|
||||
proxy_uid = {{ .Values.supervisor.sidecar.proxyUid | default 1337 }}
|
||||
process_binary_aware_network_policy = {{ .Values.supervisor.sidecar.processBinaryAwareNetworkPolicy }}
|
||||
[openshell.drivers.kubernetes.proxy_pod]
|
||||
network_policy_enforced = {{ .Values.supervisor.proxyPod.networkPolicyEnforced }}
|
||||
boundary_port = {{ .Values.supervisor.proxyPod.boundaryPort | default 5500 }}
|
||||
|
||||
{{- if not $credentialDrivers }}
|
||||
|
||||
|
||||
@@ -33,14 +33,29 @@ rules:
|
||||
- watch
|
||||
# Per-sandbox identity: TokenReview authenticates the projected token from
|
||||
# the configured sandbox service account, then the gateway resolves the
|
||||
# returned pod name and UID to the pod's `openshell.ai/sandbox-id`
|
||||
# annotation. patch is intentionally NOT granted — the annotation is set
|
||||
# once at pod create and must remain immutable for the lifetime of the
|
||||
# sandbox.
|
||||
# returned pod name and UID to its immutable sandbox labels. PATCH is used
|
||||
# only to remove the driver-owned bootstrap scheduling gate after the
|
||||
# immutable bootstrap material has been created.
|
||||
- apiGroups:
|
||||
- ""
|
||||
resources:
|
||||
- pods
|
||||
verbs:
|
||||
- get
|
||||
- patch
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["deployments"]
|
||||
verbs: ["create", "delete", "get", "list", "patch", "watch"]
|
||||
- apiGroups: ["apps"]
|
||||
resources: ["replicasets"]
|
||||
verbs: ["get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["services"]
|
||||
verbs: ["create", "delete", "get"]
|
||||
- apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create"]
|
||||
- apiGroups: ["networking.k8s.io"]
|
||||
resources: ["networkpolicies"]
|
||||
verbs: ["create", "delete", "get", "list"]
|
||||
{{- end }}
|
||||
|
||||
@@ -26,17 +26,23 @@ tests:
|
||||
resources: ["networkpolicies"]
|
||||
verbs: ["get", "create", "patch", "update"]
|
||||
|
||||
- it: omits managed NetworkPolicy permissions when isolation is disabled
|
||||
- it: preserves proxy-pod fence permissions when gateway isolation is disabled
|
||||
set:
|
||||
server.drivers.kubernetes.workspaceMode: managed
|
||||
networkPolicy.enabled: false
|
||||
asserts:
|
||||
- contains:
|
||||
path: rules
|
||||
content:
|
||||
apiGroups: ["networking.k8s.io"]
|
||||
resources: ["networkpolicies"]
|
||||
verbs: ["create", "delete", "get", "list"]
|
||||
- notContains:
|
||||
path: rules
|
||||
content:
|
||||
apiGroups: ["networking.k8s.io"]
|
||||
resources: ["networkpolicies"]
|
||||
any: true
|
||||
verbs: ["get", "create", "patch", "update"]
|
||||
|
||||
- it: grants broad secret access when credential driver is enabled (operator)
|
||||
set:
|
||||
@@ -63,17 +69,23 @@ tests:
|
||||
resourceNames: ["custom-client-tls"]
|
||||
verbs: ["get", "patch"]
|
||||
|
||||
- it: omits operator secret access when TLS and credential storage are disabled
|
||||
- it: limits operator bootstrap secret access to create when TLS and credential storage are disabled
|
||||
set:
|
||||
server.drivers.kubernetes.workspaceMode: operator
|
||||
server.disableTls: true
|
||||
asserts:
|
||||
- contains:
|
||||
path: rules
|
||||
content:
|
||||
apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create"]
|
||||
- notContains:
|
||||
path: rules
|
||||
content:
|
||||
apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
any: true
|
||||
verbs: ["get", "create", "patch", "delete"]
|
||||
|
||||
- it: restricts managed copies to TLS and configured image-pull secrets
|
||||
set:
|
||||
@@ -128,3 +140,37 @@ tests:
|
||||
apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
any: true
|
||||
|
||||
- it: grants managed proxy-pod companion permissions
|
||||
set:
|
||||
server.drivers.kubernetes.workspaceMode: managed
|
||||
asserts:
|
||||
- contains:
|
||||
path: rules
|
||||
content:
|
||||
apiGroups: ["apps"]
|
||||
resources: ["deployments"]
|
||||
verbs: ["create", "delete", "get", "list", "patch", "watch"]
|
||||
- contains:
|
||||
path: rules
|
||||
content:
|
||||
apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create"]
|
||||
|
||||
- it: grants operator proxy-pod companion permissions
|
||||
set:
|
||||
server.drivers.kubernetes.workspaceMode: operator
|
||||
asserts:
|
||||
- contains:
|
||||
path: rules
|
||||
content:
|
||||
apiGroups: ["apps"]
|
||||
resources: ["deployments"]
|
||||
verbs: ["create", "delete", "get", "list", "patch", "watch"]
|
||||
- contains:
|
||||
path: rules
|
||||
content:
|
||||
apiGroups: [""]
|
||||
resources: ["secrets"]
|
||||
verbs: ["create"]
|
||||
|
||||
@@ -237,16 +237,6 @@ tests:
|
||||
path: data["gateway.toml"]
|
||||
pattern: '(?ms)\[openshell\.gateway\][^\[]*?enable_user_namespaces'
|
||||
|
||||
- it: renders combined supervisor topology by default under [openshell.drivers.kubernetes]
|
||||
template: templates/gateway-config.yaml
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: data["gateway.toml"]
|
||||
pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"combined"'
|
||||
- notMatchRegex:
|
||||
path: data["gateway.toml"]
|
||||
pattern: 'supervisor[_]topology\s*='
|
||||
|
||||
- it: renders operator-owned upstream proxy settings under the Kubernetes driver
|
||||
template: templates/gateway-config.yaml
|
||||
set:
|
||||
@@ -312,36 +302,6 @@ tests:
|
||||
path: data["gateway.toml"]
|
||||
pattern: 'supervisor_image\s*=\s*"registry\.example\.com/openshell/supervisor:supervisor-build"'
|
||||
|
||||
- it: renders sidecar supervisor topology under [openshell.drivers.kubernetes]
|
||||
template: templates/gateway-config.yaml
|
||||
set:
|
||||
supervisor.topology: sidecar
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: data["gateway.toml"]
|
||||
pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?topology\s*=\s*"sidecar"'
|
||||
- notMatchRegex:
|
||||
path: data["gateway.toml"]
|
||||
pattern: 'supervisor[_]topology\s*='
|
||||
|
||||
- it: renders proxy uid under [openshell.drivers.kubernetes.sidecar]
|
||||
template: templates/gateway-config.yaml
|
||||
set:
|
||||
supervisor.sidecar.proxyUid: 2200
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: data["gateway.toml"]
|
||||
pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?proxy_uid\s*=\s*2200'
|
||||
|
||||
- it: renders process binary aware network policy under [openshell.drivers.kubernetes.sidecar]
|
||||
template: templates/gateway-config.yaml
|
||||
set:
|
||||
supervisor.sidecar.processBinaryAwareNetworkPolicy: false
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: data["gateway.toml"]
|
||||
pattern: '(?ms)\[openshell\.drivers\.kubernetes\.sidecar\].*?process_binary_aware_network_policy\s*=\s*false'
|
||||
|
||||
- it: configures managed SSH isolation with the gateway peer
|
||||
template: templates/gateway-config.yaml
|
||||
asserts:
|
||||
@@ -360,22 +320,6 @@ tests:
|
||||
path: data["gateway.toml"]
|
||||
pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?image_pull_secrets\s*=\s*\["regcred", "backup-regcred"\]'
|
||||
|
||||
- it: renders the default sandbox AppArmor profile under [openshell.drivers.kubernetes]
|
||||
template: templates/gateway-config.yaml
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: data["gateway.toml"]
|
||||
pattern: '(?ms)\[openshell\.drivers\.kubernetes\].*?app_armor_profile\s*=\s*"Unconfined"'
|
||||
|
||||
- it: omits sandbox AppArmor profile when disabled
|
||||
template: templates/gateway-config.yaml
|
||||
set:
|
||||
server.appArmorProfile: ""
|
||||
asserts:
|
||||
- notMatchRegex:
|
||||
path: data["gateway.toml"]
|
||||
pattern: 'app_armor_profile\s*='
|
||||
|
||||
- it: does not reuse gateway image pull secrets for sandbox pods
|
||||
template: templates/gateway-config.yaml
|
||||
set:
|
||||
|
||||
@@ -26,9 +26,8 @@ image:
|
||||
# -- Gateway image tag. Defaults to the chart appVersion when empty.
|
||||
tag: ""
|
||||
|
||||
# Supervisor image for the openshell-sandbox binary injected into sandbox pods.
|
||||
# The default repository and empty tag use the version-pinned image built into
|
||||
# the gateway. Changing the repository or setting a tag enables a Helm override.
|
||||
# Trusted runtime image. Its `openshell-sandbox` binary stages into the workload
|
||||
# runtime volume, while its `openshell-supervisor` binary runs in a separate Pod.
|
||||
supervisor:
|
||||
image:
|
||||
# -- Supervisor image repository. Changing it uses the effective gateway image tag unless tag is also set.
|
||||
@@ -39,30 +38,11 @@ supervisor:
|
||||
pullPolicy: null
|
||||
# -- Supervisor image tag override. Empty uses the version pinned into the gateway unless repository is changed.
|
||||
tag: ""
|
||||
# -- How the supervisor binary is delivered into sandbox pods.
|
||||
# Empty (default) = auto-detect from cluster version:
|
||||
# K8s >= v1.35 -> "image-volume" (ImageVolume enabled by default; GA in v1.36)
|
||||
# K8s < v1.35 -> "init-container" (copies via init container + emptyDir)
|
||||
# On K8s v1.33-v1.34 with the ImageVolume feature gate manually enabled,
|
||||
# set this to "image-volume" explicitly.
|
||||
sideloadMethod: ""
|
||||
# -- Supervisor pod topology for Kubernetes sandboxes.
|
||||
# "combined" runs the current single supervisor container in the agent pod.
|
||||
# "sidecar" runs network enforcement in a dedicated sidecar and the process
|
||||
# supervisor as a low-capability wrapper in the agent container.
|
||||
topology: "combined"
|
||||
sidecar:
|
||||
# -- UID for relaxed long-running network sidecars in sidecar topology.
|
||||
# Strict process/binary-aware sidecars run as UID 0 so Kubernetes grants
|
||||
# the required /proc inspection capabilities into the effective set. The
|
||||
# network init container installs nftables rules that exempt the effective
|
||||
# sidecar UID.
|
||||
proxyUid: 1337
|
||||
# -- Keep process/binary-aware network policy enabled in sidecar topology.
|
||||
# When false, the network sidecar runs as proxyUid, drops the extra /proc
|
||||
# inspection capabilities, and enforces endpoint/L7 policy without matching
|
||||
# policy.binaries.
|
||||
processBinaryAwareNetworkPolicy: true
|
||||
proxyPod:
|
||||
# -- Required operator acknowledgement that the cluster CNI enforces NetworkPolicy.
|
||||
networkPolicyEnforced: false
|
||||
# -- Workload boundary TLS listener port.
|
||||
boundaryPort: 5500
|
||||
|
||||
# -- Operator-owned corporate forward proxy for policy-approved TLS egress
|
||||
# from Kubernetes sandboxes. The workload cannot select or override it.
|
||||
@@ -269,12 +249,6 @@ server:
|
||||
# Linux 5.12+. When enabled, container UID 0 maps to an unprivileged host
|
||||
# UID and capabilities become namespaced.
|
||||
enableUserNamespaces: false
|
||||
# -- Kubernetes AppArmor profile requested for sandbox agent containers.
|
||||
# Default Unconfined avoids runtime/default AppArmor blocking the supervisor's
|
||||
# network namespace mount setup on AppArmor-enabled nodes. Set to "" to omit
|
||||
# the field, "RuntimeDefault" to force the runtime default profile, or
|
||||
# "Localhost/profile-name" for an operator-managed localhost profile.
|
||||
appArmorProfile: "Unconfined"
|
||||
# Kubernetes compute driver settings.
|
||||
drivers:
|
||||
kubernetes:
|
||||
|
||||
+30
-104
@@ -3,140 +3,66 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
title: "OpenShift"
|
||||
sidebar-title: "OpenShift"
|
||||
description: "Install the OpenShell Helm chart on OpenShift, including the SCC binding and chart overrides required by OpenShift's Security Context Constraints."
|
||||
description: "Install the OpenShell Helm chart on OpenShift with capability-free sandbox workloads."
|
||||
keywords: "Generative AI, Cybersecurity, Kubernetes, OpenShift, SCC, Security Context Constraints, Helm, Gateway, Installation"
|
||||
position: 6
|
||||
---
|
||||
|
||||
The Kubernetes driver resolves the UID range assigned to each OpenShift
|
||||
namespace and renders the sandbox and supervisor with a numeric non-root
|
||||
identity from that range. OpenShell does not require the `privileged` SCC or any
|
||||
added Linux capability.
|
||||
|
||||
<Warning>
|
||||
The OpenShift install path is experimental. It currently requires running sandbox pods under the `privileged` SCC and installing the gateway with TLS disabled. Use only for evaluation on a private network.
|
||||
Verify that the selected OpenShift runtime profile permits an unprivileged
|
||||
process to install a nested seccomp user-notification filter and use Landlock.
|
||||
OpenShell fails sandbox startup when either capability-free runtime probe fails.
|
||||
</Warning>
|
||||
|
||||
OpenShift's [Security Context Constraints](https://docs.openshift.com/container-platform/latest/authentication/managing-security-context-constraints.html) reject the chart's default pod security settings. Installing on OpenShift requires precreating the namespace, granting the `privileged` SCC to the sandbox service account, and overriding a few chart values so the cluster admission controller can assign UIDs and FS groups itself.
|
||||
|
||||
OpenShell installs sandbox nftables rules as individual commands. On OpenShift
|
||||
nodes where optional conntrack or packet log expressions are unavailable, those
|
||||
optional rules can fail without rolling back the required proxy bypass reject
|
||||
rules.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- OpenShift 4.x cluster with `oc` configured
|
||||
- Helm 3.x
|
||||
- [Agent Sandbox](/kubernetes/setup#install-agent-sandbox) controller and CRDs installed
|
||||
- OpenShift 4.x cluster with `oc` configured.
|
||||
- Helm 3.x.
|
||||
- [Agent Sandbox](/kubernetes/setup#install-agent-sandbox) controller and CRDs.
|
||||
- A CNI that enforces ingress and egress `NetworkPolicy` in sandbox namespaces.
|
||||
|
||||
## Install
|
||||
## Install OpenShell
|
||||
|
||||
<Steps>
|
||||
|
||||
## Create the namespace
|
||||
|
||||
Pre-create the namespace so the SCC binding can be applied before the chart installs:
|
||||
Pre-create the namespace, then install the chart. Keep the default restricted
|
||||
security posture and acknowledge NetworkPolicy only after validating the CNI.
|
||||
|
||||
```shell
|
||||
oc create ns openshell
|
||||
```
|
||||
|
||||
## Grant the privileged SCC to sandbox pods
|
||||
|
||||
Sandbox pods run under the `openshell-sandbox` service account in the `openshell` namespace and require the `privileged` SCC:
|
||||
|
||||
```shell
|
||||
oc adm policy add-scc-to-user privileged -z openshell-sandbox -n openshell
|
||||
```
|
||||
|
||||
## Install the chart with OpenShift overrides
|
||||
|
||||
```shell
|
||||
helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \
|
||||
--version <version> \
|
||||
--namespace openshell \
|
||||
--set server.disableTls=true \
|
||||
--set podSecurityContext.fsGroup=null \
|
||||
--set securityContext.runAsUser=null
|
||||
--set supervisor.proxyPod.networkPolicyEnforced=true
|
||||
```
|
||||
|
||||
| Override | Reason |
|
||||
|---|---|
|
||||
| `server.disableTls=true` | Runs the gateway over plaintext HTTP for simpler evaluation. |
|
||||
| `podSecurityContext.fsGroup=null` / `securityContext.runAsUser=null` | Clear the chart's hardcoded UID and fsGroup so OpenShift's SCC admission can assign them. |
|
||||
The driver reads the namespace's `openshift.io/sa.scc.uid-range` annotation and
|
||||
uses the resulting UID/GID for the sandbox, agent, trusted init containers, and
|
||||
supervisor. Each container sets `allowPrivilegeEscalation: false`, drops all
|
||||
Linux capabilities, and uses `RuntimeDefault` seccomp.
|
||||
|
||||
## Wait for the gateway to be ready
|
||||
Wait for the gateway:
|
||||
|
||||
```shell
|
||||
oc -n openshell rollout status statefulset/openshell
|
||||
```
|
||||
|
||||
If you set `workload.kind=deployment`, use
|
||||
`oc -n openshell rollout status deployment/openshell` instead.
|
||||
If you set `workload.kind=deployment`, wait for `deployment/openshell` instead.
|
||||
|
||||
</Steps>
|
||||
## Connect to the Gateway
|
||||
|
||||
## Connect to the gateway
|
||||
|
||||
The gateway is now running over plaintext HTTP. Connect with `oc port-forward`:
|
||||
Forward the gateway port for local evaluation:
|
||||
|
||||
```shell
|
||||
oc -n openshell port-forward svc/openshell 8080:8080
|
||||
```
|
||||
|
||||
Register the gateway with the CLI:
|
||||
|
||||
```shell
|
||||
openshell gateway add http://127.0.0.1:8080 --local --name openshift
|
||||
openshell gateway add https://127.0.0.1:8080 --local --name openshift
|
||||
openshell status
|
||||
```
|
||||
|
||||
## Production: expose externally with a real certificate
|
||||
|
||||
The steps above run the gateway over plaintext HTTP for quick evaluation. For
|
||||
a real deployment, cert-manager can issue the gateway's server certificate
|
||||
from a real Issuer or ClusterIssuer (for example, an ACME issuer), and an
|
||||
OpenShift Route with TLS passthrough exposes it externally while the gateway
|
||||
keeps terminating its own TLS and mTLS.
|
||||
|
||||
Install cert-manager and configure a working `ClusterIssuer` first — see
|
||||
[Managing Certificates](/kubernetes/managing-certificates) for the
|
||||
`certManager.serverIssuerRef` details. Configure an OIDC provider as described
|
||||
in [Access Control](/kubernetes/access-control) — remote gateways authenticate
|
||||
CLI users via OIDC, not mTLS, so the gateway must know the OIDC issuer URL.
|
||||
Install the chart with:
|
||||
|
||||
```shell
|
||||
helm install openshell oci://ghcr.io/nvidia/openshell/helm-chart \
|
||||
--version <version> \
|
||||
--namespace openshell \
|
||||
--set podSecurityContext.fsGroup=null \
|
||||
--set securityContext.runAsUser=null \
|
||||
--set server.disableTls=false \
|
||||
--set certManager.enabled=true \
|
||||
--set certManager.serverIssuerRef.name=<cluster-issuer-name> \
|
||||
--set certManager.serverIssuerRef.kind=ClusterIssuer \
|
||||
--set certManager.serverDnsNames[0]=<external-hostname> \
|
||||
--set openshiftRoute.enabled=true \
|
||||
--set openshiftRoute.host=<external-hostname> \
|
||||
--set server.oidc.issuer=<oidc-issuer-url> \
|
||||
--set server.oidc.audience=<oidc-audience>
|
||||
```
|
||||
|
||||
| Override | Reason |
|
||||
|---|---|
|
||||
| `certManager.serverIssuerRef` | Creates a second server certificate from your Issuer or ClusterIssuer for external clients. The gateway uses SNI to present this cert for the external hostname while continuing to present the internal (chart CA) cert to supervisors. The internal certificate's `ca.crt` is the chart CA that also signed the client cert, so the default `clientCaFromServerTlsSecret=true` is correct. |
|
||||
| `openshiftRoute.enabled` / `openshiftRoute.host` | Creates an OpenShift Route with TLS passthrough — the router forwards the encrypted connection by SNI without decrypting, so the gateway uses the SNI hostname to select the external certificate. |
|
||||
| `server.oidc.issuer` / `server.oidc.audience` | Configures server-side OIDC validation. Without these, the gateway expects mTLS client certificates and rejects OIDC-only CLI connections. See [Access Control](/kubernetes/access-control). |
|
||||
|
||||
Register the gateway with the CLI over OIDC. Remote gateways authenticate CLI
|
||||
users via OIDC, not mTLS — see [Access Control](/kubernetes/access-control):
|
||||
|
||||
```shell
|
||||
openshell gateway add https://<external-hostname> \
|
||||
--name openshift \
|
||||
--oidc-issuer <issuer-url>
|
||||
openshell gateway login openshift
|
||||
```
|
||||
|
||||
## Next Steps
|
||||
|
||||
- For more on certificate provisioning modes, refer to [Managing Certificates](/kubernetes/managing-certificates).
|
||||
- To expose the gateway externally through the Kubernetes Gateway API instead of a Route, refer to [Ingress](/kubernetes/ingress).
|
||||
- To configure OIDC authentication, refer to [Access Control](/kubernetes/access-control).
|
||||
For a remote deployment, configure an OpenShift Route with TLS passthrough and
|
||||
an OIDC provider. Refer to [Ingress](/kubernetes/ingress), [Managing
|
||||
Certificates](/kubernetes/managing-certificates), and [Access
|
||||
Control](/kubernetes/access-control).
|
||||
|
||||
@@ -199,14 +199,12 @@ The most commonly changed values are:
|
||||
| `server.sandboxImage` | Default sandbox image used when a sandbox does not specify one. |
|
||||
| `server.sandboxImagePullSecrets` | Image pull secrets attached to sandbox pods. Referenced Secrets must exist in the sandbox namespace. |
|
||||
| `server.grpcEndpoint` | Endpoint that sandbox supervisors use to call back to the gateway. Must be reachable from inside the cluster. |
|
||||
| `server.appArmorProfile` | AppArmor profile requested for sandbox agent containers. Defaults to `Unconfined`. |
|
||||
| `server.disableTls` | Run the gateway over plaintext HTTP. Use only behind a trusted transport. |
|
||||
| `server.auth.allowUnauthenticatedUsers` | Accept user-facing calls without OIDC or mTLS credentials. Use only for trusted local development or a fully trusted access proxy. |
|
||||
| `server.enableLoopbackServiceHttp` | Enable local plaintext HTTP for loopback sandbox service URLs. Defaults to `true`. |
|
||||
| `pkiInitJob.serverDnsNames` / `certManager.serverDnsNames` | Additional gateway server DNS SANs. Wildcard SANs also enable sandbox service URLs under that domain. |
|
||||
| `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect based on cluster version: clusters running Kubernetes 1.35 or later use `image-volume` (ImageVolume GA in 1.36); older clusters use `init-container`. Set explicitly to `image-volume` on Kubernetes 1.33 or 1.34 with the ImageVolume feature gate enabled, or to `init-container` to force the legacy path on any version. |
|
||||
| `supervisor.topology` | Sandbox pod topology. Refer to [Topology](/kubernetes/topology). |
|
||||
| `supervisor.sidecar.proxyUid` | Non-root UID used when sidecar process/binary-aware network policy is disabled. The default binary-aware sidecar runs as UID 0 instead. The configured UID must not match the sandbox UID. |
|
||||
| `supervisor.proxyPod.networkPolicyEnforced` | Required acknowledgement that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. |
|
||||
| `supervisor.proxyPod.boundaryPort` | Non-privileged TLS port used between paired supervisor and sandbox Pods. |
|
||||
| `upstreamProxy` | Operator-owned corporate HTTP forward proxy for policy-approved TLS egress. Refer to [Configure a Corporate Upstream Proxy](#configure-a-corporate-upstream-proxy). |
|
||||
|
||||
Use a values file for repeatable deployments:
|
||||
@@ -219,13 +217,6 @@ helm upgrade --install openshell \
|
||||
--values my-values.yaml
|
||||
```
|
||||
|
||||
The chart defaults `server.appArmorProfile` to `Unconfined` because
|
||||
runtime/default AppArmor profiles can block the supervisor's network namespace
|
||||
mount setup on AppArmor-enabled nodes. Set `server.appArmorProfile` to an empty
|
||||
string to omit the field, `RuntimeDefault` to force the runtime default, or
|
||||
`Localhost/<profile-name>` when you load and manage a localhost profile on each
|
||||
node.
|
||||
|
||||
To use private sandbox images, create a `kubernetes.io/dockerconfigjson` Secret
|
||||
in the sandbox namespace and reference its name:
|
||||
|
||||
@@ -265,13 +256,12 @@ upstreamProxy:
|
||||
key: credentials
|
||||
authAllowInsecure: true
|
||||
|
||||
supervisor:
|
||||
topology: sidecar
|
||||
```
|
||||
|
||||
Use `authAllowInsecure: true` only when you accept that Basic authentication is cleartext on the connection to an `http://` proxy. The initial release supports `http://` proxy endpoints and TLS CONNECT egress. It does not support HTTPS-to-proxy, custom corporate CA bundles, or forwarding plain HTTP egress through the proxy.
|
||||
|
||||
Proxy credentials require `sidecar` topology. It mounts the credential only into the dedicated network supervisor container. OpenShell rejects credential Secrets with `combined` topology because Kubernetes `fsGroup` volume permission handling can make a shared credential mount readable by the sandbox group.
|
||||
The credential mounts only in the separately scheduled supervisor Pod. The
|
||||
sandbox workload cannot read it through its environment or volumes.
|
||||
|
||||
## RBAC
|
||||
|
||||
@@ -321,7 +311,7 @@ The gateway exposes `/healthz` for process liveness and `/readyz` for dependency
|
||||
|
||||
## Next Steps
|
||||
|
||||
- To choose between combined and sidecar sandbox pods, refer to [Topology](/kubernetes/topology).
|
||||
- Kubernetes sandboxes use a separate sandbox Pod and supervisor Deployment; refer to [Topology](/kubernetes/topology).
|
||||
- To enable automatic certificate rotation with cert-manager, refer to [Managing Certificates](/kubernetes/managing-certificates).
|
||||
- To expose the gateway externally without port-forwarding, refer to [Ingress](/kubernetes/ingress).
|
||||
- To configure OIDC or reverse-proxy authentication, refer to [Access Control](/kubernetes/access-control).
|
||||
|
||||
+96
-224
@@ -3,247 +3,119 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
title: "Kubernetes Sandbox Topology"
|
||||
sidebar-title: "Topology"
|
||||
description: "Choose between combined and sidecar supervisor topology for Kubernetes sandbox pods."
|
||||
keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Sidecar, Network Policy, RuntimeClass"
|
||||
description: "Understand the capability-free sandbox and supervisor topology used for Kubernetes workloads."
|
||||
keywords: "Generative AI, Cybersecurity, Kubernetes, Sandboxing, Network Policy, Seccomp, Landlock"
|
||||
position: 2
|
||||
---
|
||||
|
||||
Kubernetes sandbox pods can run the OpenShell supervisor in `combined` or
|
||||
`sidecar` topology. Choose the topology based on which controls you need inside
|
||||
the pod and how much privilege your cluster allows on the agent container.
|
||||
OpenShell runs each Kubernetes sandbox as two separately scheduled workloads.
|
||||
The sandbox owns the agent process. The supervisor owns gateway credentials,
|
||||
policy decisions, and upstream connections.
|
||||
|
||||
## Choose a Topology
|
||||
## Understand the Components
|
||||
|
||||
The default `combined` topology preserves the full OpenShell enforcement model.
|
||||
Use `sidecar` only when you accept network-focused enforcement in exchange for a
|
||||
lower-privilege agent container.
|
||||
|
||||
| Topology | Use when | Main tradeoff |
|
||||
|---|---|---|
|
||||
| `combined` | You need OpenShell network, filesystem, and process controls in the sandbox workload. | The agent container carries the Linux capabilities the supervisor needs. |
|
||||
| `sidecar` | You need the agent container to run as non-root without added Linux capabilities, and network policy is the primary control. | Privilege-dropping and supervisor mount isolation do not run in the agent container. |
|
||||
|
||||
## Privilege Model
|
||||
|
||||
The long-running container permissions differ by topology:
|
||||
|
||||
| Topology | Pod or container | UID/GID | Privilege escalation | Capabilities | Result |
|
||||
|---|---|---|---|---|---|
|
||||
| `combined` | Agent container, which also runs the supervisor | Not forced by topology | Not explicitly disabled by the driver | Adds `SYS_ADMIN`, `NET_ADMIN`, `SYS_PTRACE`, and `SYSLOG`; adds `SETUID`, `SETGID`, and `DAC_READ_SEARCH` when user namespaces are enabled | Full supervisor controls run in the agent container. |
|
||||
| `sidecar` | Agent container, process-only supervisor (`network-only`) | `sandbox_uid:sandbox_gid` | `false` | Drops `ALL` | Agent and workload run without added Linux capabilities. |
|
||||
| `sidecar` | Network supervisor sidecar, binary-aware mode (default) | `0:sandbox_gid` | `false` | Drops `ALL`; adds `SYS_PTRACE` and `DAC_READ_SEARCH` | Root sidecar inspects cross-UID workload `/proc` entries. The nftables fence exempts UID 0, so do not inject other root containers into these pods. |
|
||||
| `sidecar` | Network supervisor sidecar, endpoint/L7-only mode | `proxyUid:sandbox_gid` | `false` | Drops `ALL` | Non-root sidecar enforces endpoint and L7 policy without matching `policy.binaries`. |
|
||||
|
||||
Short-lived setup containers still have the permissions needed to prepare the
|
||||
pod:
|
||||
|
||||
| Topology | Setup container | UID/GID | Privilege escalation | Capabilities | Purpose |
|
||||
|---|---|---|---|---|---|
|
||||
| `combined` | Supervisor install init container | `0` | Not set | Not set | Copies the supervisor binary into the agent container volume. |
|
||||
| `sidecar` | Network init container | `0` | `false` | Drops `ALL`; adds `NET_ADMIN`, `NET_RAW`, `CHOWN`, and `FOWNER` | Installs pod-local nftables rules and prepares shared sidecar state. |
|
||||
|
||||
## Combined Topology
|
||||
|
||||
Combined topology is the original Kubernetes mode and remains the default. The
|
||||
agent container starts the OpenShell supervisor, and the supervisor launches the
|
||||
workload after applying sandbox setup.
|
||||
The Kubernetes driver always uses this topology:
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
Sandbox["agents.x-k8s.io Sandbox"]
|
||||
flowchart LR
|
||||
Gateway[OpenShell gateway]
|
||||
Supervisor[Supervisor Deployment]
|
||||
Sandbox[Sandbox workload Pod]
|
||||
Agent[Agent processes]
|
||||
External[External services]
|
||||
|
||||
subgraph Pod["Sandbox pod"]
|
||||
subgraph Agent["agent container"]
|
||||
Supervisor["OpenShell supervisor<br/>network + process + filesystem"]
|
||||
Workload["Agent workload"]
|
||||
end
|
||||
end
|
||||
|
||||
Gateway["OpenShell Gateway"]
|
||||
External["External services"]
|
||||
|
||||
Sandbox --> Pod
|
||||
Supervisor --> Workload
|
||||
Supervisor -->|"gateway callback / SSH relay"| Gateway
|
||||
Supervisor -->|"policy-enforced egress"| External
|
||||
Gateway <-->|JWT-authenticated session| Supervisor
|
||||
Supervisor <-->|TLS and bootstrap token| Sandbox
|
||||
Sandbox --> Agent
|
||||
Supervisor --> External
|
||||
```
|
||||
|
||||
Combined topology keeps these controls in one supervisor path:
|
||||
`openshell-sandbox` runs as PID 1 in the workload container. It launches the
|
||||
agent, applies Landlock and child seccomp filters, identifies the process behind
|
||||
each network operation, and relays approved streams. `openshell-supervisor` runs
|
||||
in a separate Deployment. It authenticates to the gateway, evaluates policy,
|
||||
handles L7 and provider transformations, and opens upstream connections.
|
||||
|
||||
- Network endpoint and L7 policy enforcement.
|
||||
- Filesystem policy enforcement.
|
||||
- Process and binary identity checks.
|
||||
- Privilege drop into the sandbox user.
|
||||
- Gateway relay, SSH sessions, exec, and file sync.
|
||||
Both containers run as the same namespace-resolved non-root UID and GID. Their
|
||||
Pod specs set `allowPrivilegeEscalation: false`, drop every Linux capability,
|
||||
and use `RuntimeDefault` seccomp. The sandbox installs an additional nested
|
||||
seccomp user-notification filter without requesting a capability. Startup fails
|
||||
closed if the runtime blocks the required seccomp or Landlock operations.
|
||||
|
||||
Because the supervisor performs network namespace setup and process/filesystem
|
||||
controls from the agent container, Kubernetes grants that container elevated
|
||||
Linux capabilities. Use this mode when you need the complete OpenShell sandbox
|
||||
contract and your cluster policy permits those capabilities.
|
||||
## Enforce Network Isolation
|
||||
|
||||
## Sidecar Topology
|
||||
|
||||
Sidecar topology splits the supervisor into a network sidecar and a
|
||||
low-privilege process supervisor in the agent container.
|
||||
|
||||
```mermaid
|
||||
flowchart TB
|
||||
Sandbox["agents.x-k8s.io Sandbox"]
|
||||
|
||||
subgraph Pod["Sandbox pod"]
|
||||
Init["network init container<br/>root setup capabilities"]
|
||||
State["shared state + TLS volumes"]
|
||||
NetNS["pod network namespace"]
|
||||
|
||||
subgraph Agent["agent container"]
|
||||
ProcessSupervisor["process supervisor<br/>network-only"]
|
||||
Workload["Agent workload"]
|
||||
end
|
||||
|
||||
NetworkSidecar["network supervisor sidecar<br/>UID 0 by default"]
|
||||
SshEndpoint["abstract SSH relay socket<br/>peer-PID authenticated"]
|
||||
end
|
||||
|
||||
Gateway["OpenShell Gateway"]
|
||||
External["External services"]
|
||||
|
||||
Sandbox --> Pod
|
||||
Init -->|"installs nftables rules"| NetNS
|
||||
ProcessSupervisor --> Workload
|
||||
Workload -->|"egress redirected on loopback"| NetworkSidecar
|
||||
NetworkSidecar -->|"gateway session + relays"| Gateway
|
||||
NetworkSidecar -->|"policy-enforced egress"| External
|
||||
NetworkSidecar -->|"control socket + proxy TLS"| State
|
||||
ProcessSupervisor -->|"bootstrap + updates"| State
|
||||
ProcessSupervisor --> SshEndpoint
|
||||
NetworkSidecar -->|"SSH relay"| SshEndpoint
|
||||
NetworkSidecar --- State
|
||||
```
|
||||
|
||||
The pod contains these OpenShell-managed pieces:
|
||||
|
||||
| Component | Runs as | Purpose |
|
||||
|---|---|---|
|
||||
| Network init container | Root with setup capabilities | Installs pod-level nftables rules and prepares shared sidecar state. |
|
||||
| Network sidecar | UID 0 by default; `supervisor.sidecar.proxyUid` when binary-aware policy is disabled | Runs the proxy, enforces network policy, owns gateway authentication and the gateway session, and serves local policy/provider state over the sidecar control socket. |
|
||||
| Agent container | Resolved sandbox UID/GID | Runs the process supervisor and launches the user workload. |
|
||||
|
||||
In this topology, the agent container defaults to `runAsNonRoot: true`,
|
||||
`allowPrivilegeEscalation: false`, and `capabilities.drop: ["ALL"]`. The
|
||||
default binary-aware network sidecar runs as UID 0, drops default Linux
|
||||
capabilities, and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` for cross-UID workload
|
||||
process identity resolution. Setting
|
||||
`supervisor.sidecar.processBinaryAwareNetworkPolicy=false` runs the sidecar as
|
||||
the configured non-root `proxyUid`, omits both capabilities, and downgrades
|
||||
network policy to endpoint/L7 enforcement without binary matching. The root
|
||||
init container keeps the setup capabilities needed to configure pod networking.
|
||||
|
||||
Sidecar mode preserves gateway session behavior, including SSH connectivity,
|
||||
because the network sidecar owns the gateway session and bridges relay requests
|
||||
to a Linux abstract SSH socket owned by the process supervisor. The relay
|
||||
verifies the socket peer PID against the authenticated control connection, so
|
||||
the workload cannot replace the relay endpoint. The agent container does not get a
|
||||
gateway endpoint, gateway TLS material, or the sandbox bootstrap token in the
|
||||
default sidecar path.
|
||||
|
||||
<Warning>
|
||||
Sidecar mode runs the process supervisor in `network-only` mode. OpenShell still
|
||||
enforces network endpoint and L7 policy through the sidecar, and the process
|
||||
supervisor applies Landlock filesystem policy and child seccomp filters where
|
||||
the kernel/runtime supports them. The process supervisor does not perform
|
||||
root-to-sandbox privilege dropping because Kubernetes starts the container as
|
||||
the sandbox UID/GID, and it does not perform supervisor identity mount
|
||||
isolation because gateway credentials are not mounted into the agent container.
|
||||
Sidecar pods use `shareProcessNamespace: true` so the network sidecar can
|
||||
resolve workload process and binary identity through `/proc/<entrypoint-pid>`.
|
||||
</Warning>
|
||||
|
||||
## Credential Exposure
|
||||
|
||||
Sidecar topology keeps gateway credentials in the network sidecar. The agent
|
||||
container does not mount the projected ServiceAccount token used for sandbox
|
||||
token bootstrap, does not mount the sandbox client TLS secret, and does not get
|
||||
gateway callback environment variables.
|
||||
|
||||
The network sidecar serves the policy and workload-facing provider environment
|
||||
over a Unix control socket in the shared sidecar state volume. Before launching
|
||||
the workload, the process supervisor establishes the only accepted connection.
|
||||
The sidecar validates its UID, GID, and PID with peer credentials, unlinks the
|
||||
listener, derives the SSH target from trusted configuration, and rejects later
|
||||
clients. The connection receives bootstrap state and provider-environment
|
||||
updates after settings polls. If it closes, the network sidecar exits so
|
||||
Kubernetes recreates the one-client bootstrap listener, and the process
|
||||
supervisor exits so Kubernetes terminates the workload and restarts the agent
|
||||
container. This symmetric failure behavior prevents a surviving workload from
|
||||
claiming the new control listener after an isolated sidecar restart. Future
|
||||
child processes can see refreshed provider env without giving the agent
|
||||
container gateway authentication material. This does not mutate the environment
|
||||
of the already-running workload entrypoint. Use `combined` topology when you
|
||||
need the full single-supervisor enforcement path; use additional runtime
|
||||
isolation when you need a stronger container boundary around sidecar workloads.
|
||||
|
||||
## RuntimeClass Isolation
|
||||
|
||||
Sidecar topology has been validated with Kata Containers. It does not currently
|
||||
support gVisor because sidecar mode requires pod-local nftables setup, which
|
||||
gVisor does not provide to the init container. A supported sandboxed runtime
|
||||
strengthens the container boundary while OpenShell focuses on network policy
|
||||
enforcement from the sidecar.
|
||||
|
||||
Runtime classes do not re-enable the OpenShell privilege-drop or supervisor
|
||||
mount-isolation controls that sidecar mode relaxes. Use them as an additional
|
||||
workload boundary, not as a replacement for the combined topology's full
|
||||
supervisor controls.
|
||||
|
||||
You can set a default runtime class in the Kubernetes driver configuration or
|
||||
override it per sandbox with driver config:
|
||||
|
||||
```shell
|
||||
openshell sandbox create \
|
||||
--driver-config-json '{"kubernetes":{"pod":{"runtime_class_name":"kata-containers"}}}' \
|
||||
-- claude
|
||||
```
|
||||
|
||||
## Enable Sidecar Mode
|
||||
|
||||
For direct gateway TOML configuration, set the Kubernetes driver fields:
|
||||
|
||||
```toml
|
||||
[openshell.drivers.kubernetes]
|
||||
topology = "sidecar"
|
||||
|
||||
[openshell.drivers.kubernetes.sidecar]
|
||||
proxy_uid = 1337
|
||||
```
|
||||
|
||||
`proxy_uid` configures only the relaxed endpoint/L7-only sidecar. It must be at
|
||||
least `1000` and must not match the sandbox UID. The default binary-aware mode
|
||||
runs the sidecar as UID 0 instead. The network init container exempts the
|
||||
effective sidecar UID from proxy redirection so the sidecar can reach the
|
||||
gateway.
|
||||
|
||||
When the Helm chart renders `gateway.toml`, set the equivalent chart values:
|
||||
The driver creates the workload fence before it releases the workload Pod:
|
||||
|
||||
```yaml
|
||||
supervisor:
|
||||
topology: sidecar
|
||||
sidecar:
|
||||
proxyUid: 1337
|
||||
processBinaryAwareNetworkPolicy: true
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
openshell.ai/boundary-role: workload
|
||||
openshell.ai/boundary-pair: SANDBOX_ID
|
||||
policyTypes: [Ingress, Egress]
|
||||
ingress:
|
||||
- from:
|
||||
- podSelector:
|
||||
matchLabels:
|
||||
openshell.ai/boundary-role: supervisor
|
||||
openshell.ai/boundary-pair: SANDBOX_ID
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 5500
|
||||
egress: []
|
||||
```
|
||||
|
||||
Leave `topology` unset, or set it to `combined`, to keep the original
|
||||
single-container supervisor path. For Helm installs, leave
|
||||
`supervisor.topology` unset or set it to `combined`.
|
||||
The empty egress list blocks direct DNS, gateway, node, metadata, and Internet
|
||||
connections from the workload. The ingress rule permits only the paired
|
||||
supervisor to reach the sandbox TLS listener. A separate policy selects the
|
||||
supervisor and permits egress so it can reach cluster DNS, the gateway, and
|
||||
policy-approved upstream destinations.
|
||||
|
||||
Set `supervisor.sidecar.processBinaryAwareNetworkPolicy=false` only when you
|
||||
accept downgrading sidecar network policy to endpoint/L7 enforcement without
|
||||
matching `policy.binaries`. This changes the sidecar from UID 0 to `proxyUid`
|
||||
and removes its `SYS_PTRACE` and `DAC_READ_SEARCH` capabilities, which are used
|
||||
for cross-UID `/proc` inspection.
|
||||
Kubernetes policies are additive. Keep sandbox namespaces under administrative
|
||||
control so another principal cannot add permissive policies, create Pods with
|
||||
OpenShell labels, or read bootstrap Secrets. Set
|
||||
`supervisor.proxyPod.networkPolicyEnforced: true` only after you verify that the
|
||||
cluster CNI enforces both ingress and egress policies for these namespaces.
|
||||
|
||||
## Next Steps
|
||||
## Bootstrap a Sandbox
|
||||
|
||||
- To install OpenShell on Kubernetes, refer to [Setup](/kubernetes/setup).
|
||||
- To configure gateway authentication, refer to [Access Control](/kubernetes/access-control).
|
||||
- To review the driver fields, refer to [Gateway Configuration File](/reference/gateway-config).
|
||||
The driver creates each sandbox generation in a fail-closed order:
|
||||
|
||||
1. Create and validate the workload egress fence.
|
||||
2. Create the Sandbox resource with a scheduling gate.
|
||||
3. Inspect the admitted Pod identity, security context, DNS settings, and
|
||||
generation-specific Secret reference.
|
||||
4. Create separate immutable Secrets for sandbox and supervisor trust material.
|
||||
5. Start the supervisor and remove the workload scheduling gate.
|
||||
6. Publish readiness only after the supervisor attaches, confirms enforcement,
|
||||
and registers the gateway relay.
|
||||
|
||||
The trusted sandbox init container copies its Secret into a memory-backed
|
||||
volume. The main container never mounts the projected Secret and removes the
|
||||
staged bootstrap before it launches untrusted code. The supervisor receives an
|
||||
audience-bound Kubernetes token, exchanges it for a sandbox-scoped JWT, and
|
||||
keeps gateway and provider credentials outside the workload Pod.
|
||||
|
||||
Stopping a sandbox removes the workload Pod and scales the supervisor to zero.
|
||||
Starting it creates a new generation with new Secrets and a replacement
|
||||
supervisor Deployment while preserving the workspace PVC. Deletion retains the
|
||||
egress fence until Kubernetes confirms that the workload Pod is gone.
|
||||
|
||||
## Check Cluster Requirements
|
||||
|
||||
The topology requires the following cluster behavior:
|
||||
|
||||
- Linux nodes and a container runtime that permits an unprivileged process to
|
||||
install a nested seccomp user-notification filter under `RuntimeDefault`.
|
||||
- Landlock enabled and usable by the non-root sandbox process.
|
||||
- A CNI that enforces `networking.k8s.io/v1` ingress and egress policies,
|
||||
including node-local and metadata destinations.
|
||||
- Support for Pod scheduling gates and the safe
|
||||
`net.ipv4.ip_unprivileged_port_start=0` sysctl.
|
||||
- Administrative control of sandbox namespaces and OpenShell pair labels.
|
||||
|
||||
OpenShell actively probes the Linux primitives and validates the admitted Pod
|
||||
before starting the agent. Treat a failed probe or changed security posture as
|
||||
an unsupported runtime, not a degraded mode.
|
||||
|
||||
@@ -546,13 +546,13 @@ image_pull_secrets = ["regcred"]
|
||||
# supervisor_image = "ghcr.io/nvidia/openshell/supervisor:<version>"
|
||||
supervisor_image_pull_policy = "if_not_present"
|
||||
|
||||
# Use the image volume on Kubernetes >= 1.35 (GA in 1.36); switch to "init-container"
|
||||
# on older clusters or where the ImageVolume feature gate is off.
|
||||
supervisor_sideload_method = "image-volume"
|
||||
# "combined" runs the existing single supervisor container with full process,
|
||||
# filesystem, and network enforcement in the agent container. "sidecar" moves
|
||||
# pod-level network enforcement and gateway session handling into a network sidecar.
|
||||
topology = "combined"
|
||||
[openshell.drivers.kubernetes.proxy_pod]
|
||||
# Set this only after verifying that the cluster CNI enforces both ingress and
|
||||
# egress NetworkPolicy in every sandbox namespace.
|
||||
network_policy_enforced = true
|
||||
boundary_port = 5500
|
||||
|
||||
[openshell.drivers.kubernetes]
|
||||
# Optional corporate HTTP forward proxy for policy-approved TLS egress. The
|
||||
# sandbox workload cannot select or override these settings. Only http:// proxy
|
||||
# endpoints and TLS CONNECT traffic are supported; plain HTTP egress remains
|
||||
@@ -570,9 +570,7 @@ topology = "combined"
|
||||
# Secret prevents that Pod from starting; unreadable or malformed `user:pass`
|
||||
# content is validated fail-closed by the supervisor at startup and never
|
||||
# falls back to direct egress.
|
||||
# Proxy credential Secrets require `topology = "sidecar"`. Combined topology
|
||||
# shares its credential mount with the workload and can make it readable by the
|
||||
# sandbox group through Kubernetes `fsGroup` volume permission handling.
|
||||
# Proxy credentials mount only in the separately scheduled supervisor Pod.
|
||||
# Required with a credential Secret: Basic authentication to an http:// proxy
|
||||
# is cleartext on the connection to that proxy.
|
||||
# proxy_auth_allow_insecure = true
|
||||
@@ -587,7 +585,6 @@ ssh_socket_path = "/run/openshell/ssh.sock"
|
||||
client_tls_secret_name = "openshell-client-tls"
|
||||
host_gateway_ip = "10.0.0.1"
|
||||
enable_user_namespaces = false
|
||||
app_armor_profile = "Unconfined"
|
||||
workspace_default_storage_size = "10Gi"
|
||||
# Kubernetes StorageClass for the workspace PVC. Empty (default) omits the
|
||||
# field, using the cluster's default StorageClass. Set this on clusters with no
|
||||
@@ -622,19 +619,12 @@ enabled = true
|
||||
gateway_namespace = "openshell"
|
||||
gateway_pod_selector = { "app.kubernetes.io/name" = "openshell", "app.kubernetes.io/instance" = "openshell" }
|
||||
|
||||
[openshell.drivers.kubernetes.sidecar]
|
||||
# UID used by relaxed long-running network sidecars. Strict process/binary-aware
|
||||
# sidecars run as UID 0 so Kubernetes grants the required /proc inspection
|
||||
# capabilities into the effective set. In sidecar topology the network init
|
||||
# container installs nftables rules that exempt the effective sidecar UID, so
|
||||
# this dedicated infrastructure UID must remain at least 1000 and must not
|
||||
# match the sandbox workload UID.
|
||||
proxy_uid = 1337
|
||||
# Keep process/binary-aware network policy enabled in sidecar topology. Set
|
||||
# false to run the sidecar as proxy_uid, drop the sidecar's extra /proc
|
||||
# inspection capabilities, and enforce endpoint/L7 policy without matching
|
||||
# policy.binaries.
|
||||
process_binary_aware_network_policy = true
|
||||
[openshell.drivers.kubernetes.proxy_pod]
|
||||
# Required acknowledgement that the cluster CNI enforces NetworkPolicy and the
|
||||
# sandbox namespaces prevent untrusted policy, pod, label, and Secret changes.
|
||||
network_policy_enforced = true
|
||||
# TLS-protected boundary listener reached only by the paired control pod.
|
||||
boundary_port = 5500
|
||||
```
|
||||
|
||||
In managed workspace mode, the Kubernetes driver copies each explicitly named
|
||||
|
||||
@@ -242,7 +242,7 @@ OpenShell rejects mount `source`, `target`, and Docker volume `subpath` values
|
||||
with surrounding whitespace. OpenShell also rejects mount targets that replace
|
||||
the workspace root or container root, or contain or are contained by the
|
||||
configured SSH socket or reserved `/opt/openshell`, `/etc/openshell`,
|
||||
`/etc/openshell-tls`, `/run/openshell`, `/run/openshell-sidecar`, and network
|
||||
`/etc/openshell-tls`, `/run/openshell`, and network
|
||||
namespace roots. These checks do not make host bind mounts safe.
|
||||
|
||||
## Podman Driver
|
||||
@@ -421,19 +421,16 @@ For maintainer-level implementation details, refer to the [Kubernetes driver REA
|
||||
| `[managed_ssh_ingress]` | `networkPolicy.enabled` | In managed mode, create an SSH ingress policy in every workspace namespace. Helm configures the gateway namespace and pod selector automatically. Operator mode leaves namespace policy management to the platform operator. |
|
||||
| `grpc_endpoint` | `server.grpcEndpoint` | Set the gateway callback endpoint reachable from sandbox pods. |
|
||||
| `client_tls_secret_name` | `server.tls.clientTlsSecretName` | Mount sandbox client TLS materials from a Kubernetes secret. |
|
||||
| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the supervisor image that provides the `openshell-sandbox` binary. The default repository with an empty tag uses the version-pinned image built into the gateway. Changing the repository uses the effective gateway image tag, while setting a tag pins that version explicitly. |
|
||||
| `supervisor_image` | `supervisor.image.repository` / `supervisor.image.tag` | Override the trusted runtime image that provides the `openshell-sandbox` and `openshell-supervisor` binaries. The default repository with an empty tag uses the version pinned into the gateway. |
|
||||
| `supervisor_image_pull_policy` | `supervisor.image.pullPolicy` | Set the Kubernetes image pull policy for the supervisor image. |
|
||||
| `supervisor_sideload_method` | `supervisor.sideloadMethod` | How the supervisor binary is delivered into sandbox pods. Leave empty to auto-detect from cluster version. Set to `image-volume` to mount the supervisor OCI image directly as a volume (requires Kubernetes 1.33+ with the ImageVolume feature gate; GA in 1.36), or `init-container` to copy it through an init container on older clusters. |
|
||||
| `topology` | `supervisor.topology` | Set `combined` for the default single supervisor path, or `sidecar` to move pod-level network enforcement and the gateway session into a dedicated sidecar. |
|
||||
| `proxy_pod.network_policy_enforced` | `supervisor.proxyPod.networkPolicyEnforced` | Acknowledge that the cluster CNI enforces ingress and egress `NetworkPolicy` in sandbox namespaces. This must be `true`. |
|
||||
| `proxy_pod.boundary_port` | `supervisor.proxyPod.boundaryPort` | Set the non-privileged TLS port used between the paired supervisor and sandbox Pods. |
|
||||
| `https_proxy` | `upstreamProxy.url` | Set the operator-owned `http://host:port` corporate forward proxy used for policy-approved TLS CONNECT egress. |
|
||||
| `no_proxy` | `upstreamProxy.noProxy` | Set destinations that bypass only the corporate proxy. OpenShell policy evaluation still applies. |
|
||||
| `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. Requires `sidecar` topology. |
|
||||
| `proxy_auth_secret_key` | `upstreamProxy.authSecret.key` | Set the Secret key containing the `user:pass` credential. Requires `sidecar` topology. |
|
||||
| `proxy_auth_secret_name` | `upstreamProxy.authSecret.name` | Set the existing Secret name in the sandbox namespace that contains the proxy credential. The Secret mounts only in the supervisor Pod. |
|
||||
| `proxy_auth_secret_key` | `upstreamProxy.authSecret.key` | Set the Secret key containing the `user:pass` credential. |
|
||||
| `proxy_auth_allow_insecure` | `upstreamProxy.authAllowInsecure` | Set `true` to acknowledge that Basic authentication to an HTTP proxy is cleartext. Required with a proxy credential Secret. |
|
||||
| `proxy_connect_by_hostname` | `upstreamProxy.connectByHostname` | Send hostnames rather than validated IPs in CONNECT requests. Use only when proxy ACLs require hostname targets. |
|
||||
| `sidecar.proxy_uid` | `supervisor.sidecar.proxyUid` | Dedicated UID of at least `1000` used by the relaxed sidecar when process/binary-aware network policy is disabled. It must not match the workload UID. The default binary-aware sidecar runs as UID 0. The network init container exempts the effective sidecar UID from proxy redirection. |
|
||||
| `sidecar.process_binary_aware_network_policy` | `supervisor.sidecar.processBinaryAwareNetworkPolicy` | Keep process/binary-aware network policy enabled in `sidecar` topology. The default runs the sidecar as UID 0 with `SYS_PTRACE` and `DAC_READ_SEARCH`. Set false to run as `proxy_uid`, drop both capabilities, and enforce endpoint/L7 policy without matching `policy.binaries`. |
|
||||
| `app_armor_profile` | `server.appArmorProfile` | Set the sandbox agent container's AppArmor profile. Helm defaults this to `Unconfined` so AppArmor-enabled nodes do not block supervisor network namespace setup. Set the Helm value to an empty string to omit the field, or use `RuntimeDefault` or `Localhost/<profile-name>` for operator-managed profiles. |
|
||||
| `workspace_default_storage_size` | `server.workspaceDefaultStorageSize` | Set the default workspace PVC size for new sandboxes. |
|
||||
| `workspace_storage_class` | `server.workspaceStorageClass` | Set the `StorageClass` for the workspace PVC. Empty (default) omits `storageClassName` and uses the cluster's default `StorageClass`. Set this on clusters with no default `StorageClass`, otherwise the workspace PVC stays `Pending` and the sandbox never starts. |
|
||||
| `sa_token_ttl_secs` | `server.sandboxJwt.k8sSaTokenTtlSecs` | Set the projected ServiceAccount token TTL used for the bootstrap token exchange. |
|
||||
@@ -445,30 +442,15 @@ remain limited to the explicitly configured TLS and image-pull Secret names.
|
||||
The driver creates copies only in gateway-owned managed namespaces. Do not
|
||||
reuse the gateway ServiceAccount for unrelated workloads.
|
||||
|
||||
In `combined` topology, the agent container carries the Linux capabilities
|
||||
needed by the supervisor for network namespace setup, Landlock filesystem
|
||||
policy, process privilege changes, and network policy enforcement. In `sidecar`
|
||||
topology, the agent container runs as the resolved sandbox UID/GID with no added
|
||||
Linux capabilities. A root init container performs the nftables setup, and the
|
||||
long-running binary-aware sidecar runs as UID 0, drops default capabilities,
|
||||
and adds `SYS_PTRACE` plus `DAC_READ_SEARCH` for workload process identity
|
||||
resolution through shared `/proc`. The
|
||||
`sidecar.process_binary_aware_network_policy = false` setting runs it as the
|
||||
configured non-root `proxy_uid`, removes both capabilities, and relaxes network
|
||||
policy to endpoint/L7 matching only. The
|
||||
network sidecar owns gateway authentication and writes local policy/provider
|
||||
state to the process supervisor over a local control socket, so the agent
|
||||
container does not mount the sandbox bootstrap token or client TLS secret in
|
||||
the default sidecar path. The provider environment is refreshed by the network
|
||||
sidecar after settings polls and streamed to the process supervisor so future
|
||||
child processes can see updated provider env without gateway access in the
|
||||
agent container.
|
||||
Sidecar mode keeps gateway session and SSH behavior. The process supervisor
|
||||
applies Landlock filesystem policy and child seccomp filters where supported,
|
||||
but it does not perform root-to-sandbox privilege dropping or supervisor
|
||||
identity mount isolation. Network policy still runs in the sidecar, and sidecar
|
||||
pods set `shareProcessNamespace: true` so the network sidecar can resolve
|
||||
process/binary identity through `/proc/<entrypoint-pid>`.
|
||||
The Kubernetes driver always uses a capability-free proxy-pod topology. The
|
||||
workload Pod runs `openshell-sandbox` as the same non-root UID/GID as the agent
|
||||
and requests no added Linux capabilities. A separate Deployment runs
|
||||
`openshell-supervisor`, authenticates to the gateway with a sandbox JWT, and
|
||||
owns upstream connections. Both containers disable privilege escalation, drop
|
||||
all capabilities, and use `RuntimeDefault` seccomp. The sandbox adds a nested
|
||||
seccomp user-notification filter and Landlock restrictions before it launches
|
||||
the agent. An empty-egress `NetworkPolicy` is the mandatory outer fence for the
|
||||
workload Pod.
|
||||
|
||||
The Kubernetes driver creates namespaced `agents.x-k8s.io` `Sandbox` resources from the Kubernetes SIG Apps [agent-sandbox](https://github.com/kubernetes-sigs/agent-sandbox) project. It detects the served Sandbox API at runtime, caches the selected API version for the gateway process, and uses `v1beta1` when available before falling back to `v1alpha1`, so supported Agent Sandbox installations work without version-specific operator configuration. The Agent Sandbox controller turns those resources into sandbox pods and related storage.
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ ENV OPENSHELL_COMPUTE_DRIVER=kubernetes \
|
||||
OPENSHELL_SUPERVISOR_IMAGE=${SUPERVISOR_IMAGE} \
|
||||
OPENSHELL_SUPERVISOR_IMAGE_PULL_POLICY=if_not_present \
|
||||
OPENSHELL_SUPERVISOR_SIDELOAD_METHOD=init-container \
|
||||
OPENSHELL_K8S_TOPOLOGY=combined
|
||||
OPENSHELL_K8S_PROXY_POD_NETWORK_POLICY_ENFORCED=true
|
||||
|
||||
USER 1000:1000
|
||||
EXPOSE 8080
|
||||
|
||||
@@ -3,7 +3,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
# Helm post-renderer for the external Kubernetes compute-driver smoke test.
|
||||
# It keeps the test-only sidecar and Unix socket plumbing out of the chart.
|
||||
# It keeps the test-only external-driver container and Unix socket plumbing out
|
||||
# of the chart.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
//! is dropped, replacing the `trap cleanup EXIT` pattern from the bash tests.
|
||||
|
||||
use std::process::Stdio;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
@@ -31,6 +32,23 @@ fn extract_sandbox_name(output: &str) -> Option<String> {
|
||||
/// startup.
|
||||
const SANDBOX_READY_TIMEOUT: Duration = Duration::from_secs(600);
|
||||
|
||||
static NEXT_SANDBOX_NAME: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
fn has_explicit_sandbox_name(args: &[&str]) -> bool {
|
||||
args.iter()
|
||||
.any(|arg| *arg == "--name" || arg.starts_with("--name="))
|
||||
}
|
||||
|
||||
fn add_unique_name_if_missing(command: &mut tokio::process::Command, args: &[&str]) {
|
||||
if !has_explicit_sandbox_name(args) {
|
||||
command.arg("--name").arg(format!(
|
||||
"e2e-{}-{}",
|
||||
std::process::id(),
|
||||
NEXT_SANDBOX_NAME.fetch_add(1, Ordering::Relaxed)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
/// RAII guard that deletes a sandbox on drop.
|
||||
///
|
||||
/// For sandboxes created with `--keep` (long-running background command), the
|
||||
@@ -92,6 +110,7 @@ impl SandboxGuard {
|
||||
|
||||
let mut cmd = openshell_cmd();
|
||||
cmd.arg("sandbox").arg("create").arg("--detach");
|
||||
add_unique_name_if_missing(&mut cmd, create_args);
|
||||
for arg in create_args {
|
||||
cmd.arg(arg);
|
||||
}
|
||||
@@ -166,10 +185,9 @@ impl SandboxGuard {
|
||||
/// which lets tests control competing and reconnecting clients directly.
|
||||
pub async fn create_detached_main(command: &[&str]) -> Result<Self, String> {
|
||||
let mut cmd = openshell_cmd();
|
||||
cmd.arg("sandbox")
|
||||
.arg("create")
|
||||
.arg("--detach")
|
||||
.arg("--")
|
||||
cmd.arg("sandbox").arg("create").arg("--detach");
|
||||
add_unique_name_if_missing(&mut cmd, &[]);
|
||||
cmd.arg("--")
|
||||
.args(command)
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped());
|
||||
@@ -216,6 +234,7 @@ impl SandboxGuard {
|
||||
) -> Result<Self, String> {
|
||||
let mut create_cmd = openshell_cmd();
|
||||
create_cmd.arg("sandbox").arg("create").arg("--detach");
|
||||
add_unique_name_if_missing(&mut create_cmd, create_args);
|
||||
for arg in create_args {
|
||||
create_cmd.arg(arg);
|
||||
}
|
||||
@@ -354,6 +373,7 @@ impl SandboxGuard {
|
||||
) -> Result<Self, String> {
|
||||
let mut cmd = openshell_cmd();
|
||||
cmd.arg("sandbox").arg("create").arg("--detach");
|
||||
add_unique_name_if_missing(&mut cmd, &[]);
|
||||
for (local, dest) in uploads {
|
||||
cmd.arg("--upload").arg(format!("{local}:{dest}"));
|
||||
}
|
||||
@@ -680,3 +700,15 @@ impl Drop for SandboxGuard {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::has_explicit_sandbox_name;
|
||||
|
||||
#[test]
|
||||
fn detects_explicit_sandbox_names() {
|
||||
assert!(has_explicit_sandbox_name(&["--name", "example"]));
|
||||
assert!(has_explicit_sandbox_name(&["--name=example"]));
|
||||
assert!(!has_explicit_sandbox_name(&["--policy", "policy.yaml"]));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
//! Kubernetes wiring coverage for authenticated corporate forward proxies.
|
||||
//!
|
||||
//! The shell wrapper configures the gateway before it starts, creates the
|
||||
//! credential Secret, and forces sidecar topology. This test starts a proxy
|
||||
//! credential Secret, and enables proxy-pod topology. This test starts a proxy
|
||||
//! and an HTTPS upstream on the host visible to sandbox pods, then proves that
|
||||
//! a permitted request uses authenticated CONNECT while a policy-denied port
|
||||
//! never reaches the proxy.
|
||||
|
||||
@@ -115,23 +115,26 @@ async fn delete_sandbox(name: &str) {
|
||||
#[tokio::test]
|
||||
#[allow(clippy::too_many_lines)] // end-to-end test exercises full label lifecycle
|
||||
async fn sandbox_labels_are_stored_and_filterable() {
|
||||
// Keep the unique names below the public 19-character routable-name
|
||||
// limit. Hex keeps the per-process suffix compact on busy CI hosts.
|
||||
let suffix = format!("{:x}", std::process::id());
|
||||
let dev_backend = format!("lbl-db-{suffix}");
|
||||
let staging_backend = format!("lbl-sb-{suffix}");
|
||||
let prod_frontend = format!("lbl-pf-{suffix}");
|
||||
let dev_data = format!("lbl-dd-{suffix}");
|
||||
|
||||
// Create sandboxes with different labels
|
||||
let name1 =
|
||||
create_sandbox_with_labels("e2e-lbl-dev-back", &[("env", "dev"), ("team", "backend")])
|
||||
.await;
|
||||
create_sandbox_with_labels(&dev_backend, &[("env", "dev"), ("team", "backend")]).await;
|
||||
|
||||
let name2 = create_sandbox_with_labels(
|
||||
"e2e-lbl-stg-back",
|
||||
&[("env", "staging"), ("team", "backend")],
|
||||
)
|
||||
.await;
|
||||
let name2 =
|
||||
create_sandbox_with_labels(&staging_backend, &[("env", "staging"), ("team", "backend")])
|
||||
.await;
|
||||
|
||||
let name3 =
|
||||
create_sandbox_with_labels("e2e-lbl-prd-frnt", &[("env", "prod"), ("team", "frontend")])
|
||||
.await;
|
||||
create_sandbox_with_labels(&prod_frontend, &[("env", "prod"), ("team", "frontend")]).await;
|
||||
|
||||
let name4 =
|
||||
create_sandbox_with_labels("e2e-lbl-dev-data", &[("env", "dev"), ("team", "data")]).await;
|
||||
let name4 = create_sandbox_with_labels(&dev_data, &[("env", "dev"), ("team", "data")]).await;
|
||||
|
||||
// Test 1: Verify labels are stored in sandbox metadata
|
||||
let details = get_sandbox_details(&name1).await;
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
//! Enables `OPENSHELL_ENABLE_USER_NAMESPACES` on the gateway, triggers sandbox
|
||||
//! creation, and inspects the resulting pod spec to confirm:
|
||||
//! 1. `spec.hostUsers` is `false`
|
||||
//! 2. The container security context includes the extra capabilities
|
||||
//! (SETUID, SETGID, DAC_READ_SEARCH) required for user namespace operation
|
||||
//! 2. The container security context requests no added capabilities and
|
||||
//! drops every capability
|
||||
//!
|
||||
//! The sandbox pod may fail to start in Docker-in-Docker dev clusters where the
|
||||
//! filesystem does not support ID-mapped mounts. The test inspects the pod spec
|
||||
@@ -211,7 +211,7 @@ async fn sandbox_pod_spec_has_user_namespace_fields() {
|
||||
.await;
|
||||
|
||||
// Inspect capabilities on the agent container.
|
||||
let caps = kubectl(&[
|
||||
let cap_add = kubectl(&[
|
||||
"get",
|
||||
"pod",
|
||||
&sandbox_name,
|
||||
@@ -221,6 +221,16 @@ async fn sandbox_pod_spec_has_user_namespace_fields() {
|
||||
"jsonpath={.spec.containers[?(@.name=='agent')].securityContext.capabilities.add}",
|
||||
])
|
||||
.await;
|
||||
let cap_drop = kubectl(&[
|
||||
"get",
|
||||
"pod",
|
||||
&sandbox_name,
|
||||
"-n",
|
||||
"openshell",
|
||||
"-o",
|
||||
"jsonpath={.spec.containers[?(@.name=='agent')].securityContext.capabilities.drop}",
|
||||
])
|
||||
.await;
|
||||
|
||||
// Clean up.
|
||||
stop_child(&mut child).await;
|
||||
@@ -235,18 +245,14 @@ async fn sandbox_pod_spec_has_user_namespace_fields() {
|
||||
"sandbox pod must have spec.hostUsers=false when user namespaces are enabled"
|
||||
);
|
||||
|
||||
// Assert extra capabilities are present.
|
||||
let caps_val = caps.expect("failed to get capabilities from pod spec");
|
||||
for cap in ["SETUID", "SETGID", "DAC_READ_SEARCH"] {
|
||||
assert!(
|
||||
caps_val.contains(cap),
|
||||
"sandbox pod must include {cap} in capabilities when user namespaces are enabled, got: {caps_val}"
|
||||
);
|
||||
}
|
||||
for cap in ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE", "SYSLOG"] {
|
||||
assert!(
|
||||
caps_val.contains(cap),
|
||||
"sandbox pod must include {cap} in capabilities, got: {caps_val}"
|
||||
);
|
||||
}
|
||||
let cap_add = cap_add.expect("failed to get added capabilities from pod spec");
|
||||
assert!(
|
||||
cap_add.trim().is_empty(),
|
||||
"sandbox pod must not request added capabilities, got: {cap_add}"
|
||||
);
|
||||
let cap_drop = cap_drop.expect("failed to get dropped capabilities from pod spec");
|
||||
assert!(
|
||||
cap_drop.contains("ALL"),
|
||||
"sandbox pod must drop every capability, got: {cap_drop}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -258,9 +258,34 @@ cleanup() {
|
||||
&& kctl get namespace "${NAMESPACE}" >/dev/null 2>&1; then
|
||||
echo "=== gateway pod state (preserved for debugging) ==="
|
||||
kctl -n "${NAMESPACE}" get pods -o wide 2>&1 || true
|
||||
echo "=== Agent Sandbox resources ==="
|
||||
kctl -n "${NAMESPACE}" get sandboxes.agents.x-k8s.io -o yaml 2>&1 || true
|
||||
echo "=== gateway sandbox records ==="
|
||||
"${OPENSHELL_BIN:-${ROOT}/target/debug/openshell}" \
|
||||
sandbox list --all-workspaces --output json 2>&1 || true
|
||||
echo "=== proxy-pod supervisor Deployments ==="
|
||||
kctl -n "${NAMESPACE}" get deployments \
|
||||
-l "openshell.ai/boundary-role=supervisor" -o yaml 2>&1 || true
|
||||
echo "=== proxy-pod supervisor logs (last 200 lines each) ==="
|
||||
while IFS= read -r supervisor_pod; do
|
||||
[ -n "${supervisor_pod}" ] || continue
|
||||
echo "--- ${supervisor_pod} ---"
|
||||
kctl -n "${NAMESPACE}" logs "${supervisor_pod}" \
|
||||
--all-containers --prefix --tail=200 2>&1 || true
|
||||
echo "--- ${supervisor_pod} (previous containers) ---"
|
||||
kctl -n "${NAMESPACE}" logs "${supervisor_pod}" --previous \
|
||||
--all-containers --prefix --tail=200 2>&1 || true
|
||||
done < <(kctl -n "${NAMESPACE}" get pods \
|
||||
-l "openshell.ai/boundary-role=supervisor" -o name 2>/dev/null || true)
|
||||
echo "=== gateway events ==="
|
||||
kctl -n "${NAMESPACE}" get events --sort-by=.lastTimestamp 2>&1 \
|
||||
| tail -n 80 || true
|
||||
echo "=== gateway lifecycle and supervisor-session logs ==="
|
||||
kctl -n "${NAMESPACE}" logs "$(kube_workload_ref "${RELEASE_NAME}")" \
|
||||
--since=20m \
|
||||
--all-containers --prefix 2>&1 \
|
||||
| grep -Ei "sandbox phase changed|start_sandbox|stop_sandbox|supervisor session|proxy-pod|bootstrap" \
|
||||
|| true
|
||||
echo "=== gateway logs (last 200 lines) ==="
|
||||
kctl -n "${NAMESPACE}" logs \
|
||||
-l "app.kubernetes.io/instance=${RELEASE_NAME}" --tail=200 \
|
||||
|
||||
@@ -223,7 +223,8 @@ Common findings:
|
||||
callbacks. On an older release, set `bind_address = "127.0.0.1:17670"` or
|
||||
upgrade.
|
||||
- Sandbox runtime image exits before printing `openshell-sandbox --version`: verify the configured image contains a static executable at `/openshell-sandbox`.
|
||||
- A sandbox with explicit `protocol: tcp` endpoints fails before workload readiness: confirm the Docker or Podman driver supplied the `policy-dns-transparent-tcp` runtime capability and inspect supervisor logs for missing `nft`, synthetic-route overlap, or namespace-local DNS/TCP listener bind failures. Kubernetes, VM, sidecar, and out-of-tree drivers must reject this policy until they provide the complete substrate; use omitted protocol with an explicit proxy on those runtimes.
|
||||
- Supervisor runtime validation fails: verify `supervisor_image` contains a static `/openshell-supervisor` executable from the same release as the sandbox runtime.
|
||||
- The sandbox fails its enforcement probe: inspect the sandbox log for the exact nested seccomp user-notification, task-memory, Landlock, loopback DNS, or socket-injection check that failed. Do not add capabilities or switch to an unconfined seccomp profile; use a runtime whose default profile permits the unprivileged probe.
|
||||
- A GPU sandbox fails because Docker reports no discovered NVIDIA CDI devices: verify `.DiscoveredDevices` contains entries such as `nvidia.com/gpu=all`, verify `/etc/cdi` or `/var/run/cdi` contains a generated NVIDIA spec, and check that `nvidia-cdi-refresh.service` and `nvidia-cdi-refresh.path` from NVIDIA Container Toolkit are enabled and healthy. The service is a one-shot unit, so `inactive (dead)` can be normal after a successful run; use `systemctl status` and `journalctl` to distinguish success from a skipped or failed refresh. Restart `nvidia-cdi-refresh.service` to regenerate missing or stale CDI specs, then restart or reload Docker and re-check `docker info`.
|
||||
|
||||
During a graceful gateway restart, Docker, Podman, and VM sandboxes with
|
||||
@@ -426,7 +427,7 @@ If `server.providerTokenGrants.spiffe.enabled=true`, the gateway should still
|
||||
render `[openshell.gateway.gateway_jwt]` and mount the `sandbox-jwt` Secret.
|
||||
SPIRE is used by both the gateway and sandbox supervisors for dynamic provider
|
||||
token grants. The gateway pod must mount the `spiffe-workload-api` CSI volume
|
||||
and set `OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`; sandbox pods must
|
||||
and set `OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET`; supervisor Pods must
|
||||
receive the matching Workload API socket from the Kubernetes driver config.
|
||||
The gateway verifies supervisor JWT-SVIDs from JWT bundles fetched through this
|
||||
Workload API socket, not from the SPIRE OIDC discovery endpoint.
|
||||
@@ -549,59 +550,47 @@ kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\
|
||||
kubectl -n <sandbox-namespace> get sandbox <sandbox-name> -o jsonpath='{.spec.template.spec.serviceAccountName}{"\n"}'
|
||||
```
|
||||
|
||||
If `topology = "sidecar"` is rendered under `[openshell.drivers.kubernetes]`,
|
||||
sandbox pods should have an `openshell-network-init` init container running
|
||||
`--mode=network-init`, an `agent` container running
|
||||
`openshell-sandbox --mode=process`, and an `openshell-supervisor-network`
|
||||
container running `--mode=network`. The init container owns nftables setup and
|
||||
should be the only sidecar topology container with `NET_ADMIN`. It also needs
|
||||
`CHOWN`/`FOWNER` to hand shared emptyDir state to the effective sidecar UID. The
|
||||
default binary-aware network sidecar runs as UID 0 with primary GID
|
||||
`sandbox_gid` and adds `SYS_PTRACE` plus `DAC_READ_SEARCH`. When
|
||||
`process_binary_aware_network_policy = false`, it runs as the configured
|
||||
non-root `proxy_uid` without those inspection capabilities. That dedicated
|
||||
proxy UID must remain at least `1000` and must not match the workload UID
|
||||
because the pod egress fence exempts its traffic. The pod `fsGroup` is set to
|
||||
`sandbox_gid` in both modes.
|
||||
Kubernetes always uses the proxy-pod topology. Helm must render
|
||||
`network_policy_enforced = true`. This is an explicit operator assertion that
|
||||
the cluster CNI enforces Kubernetes NetworkPolicy; the Kubernetes API cannot
|
||||
attest enforcement. Run sandboxes only in a trusted namespace
|
||||
where tenants cannot create pods, copy the OpenShell pair labels, or read the
|
||||
bootstrap Secret.
|
||||
|
||||
In sidecar topology only the network sidecar should mount the gateway bootstrap
|
||||
credentials (`openshell-sa-token` and `openshell-client-tls`). The process
|
||||
container should not receive `OPENSHELL_ENDPOINT`, gateway TLS env vars, the
|
||||
sandbox token file, or those credential mounts. Instead, the network sidecar
|
||||
serves policy and provider environment state over the Unix control socket from
|
||||
`OPENSHELL_SIDECAR_CONTROL_SOCKET` (`/run/openshell-sidecar/control.sock` by
|
||||
default). The process supervisor must be the first and only client. After
|
||||
validating its peer UID, GID, and PID, the sidecar unlinks the listener. If the
|
||||
connection later closes, the network sidecar exits non-zero so Kubernetes can
|
||||
restart it with a fresh listener. If the process supervisor fails before
|
||||
launching the workload,
|
||||
inspect both containers for control-socket bind, connect, bootstrap, or update
|
||||
errors. If new SSH/exec sessions do not pick up refreshed provider environment,
|
||||
inspect the network sidecar settings-poll logs and the process container logs
|
||||
for provider environment update handling; the process container should consume
|
||||
newer provider-env revisions without receiving gateway credentials.
|
||||
The workload Pod runs `/openshell-sandbox`. It has no
|
||||
gateway credentials and no direct egress. A deny-all workload NetworkPolicy is
|
||||
created before the suspended Sandbox resource, then a per-sandbox Service,
|
||||
split immutable bootstrap Secrets, supervisor Deployment, supervisor-egress
|
||||
policy, and sandbox-ingress policy are provisioned before the workload is
|
||||
released. The supervisor Pod runs `/openshell-supervisor`. Both Pods use the
|
||||
same resolved non-root identity, request no capabilities, drop `ALL`, disable
|
||||
privilege escalation, and use `RuntimeDefault` seccomp. The supervisor reaches
|
||||
the sandbox over per-sandbox TLS with server-certificate verification plus
|
||||
bootstrap-token client authentication, and owns gateway policy, provider
|
||||
credentials, DNS, and mediated upstream connections.
|
||||
|
||||
The process container reports the workload entrypoint PID over the same control
|
||||
socket, and the network sidecar uses that PID for binary-scoped policy
|
||||
decisions through `/proc`. If rules with `policy.binaries` are unexpectedly
|
||||
denied, inspect the sidecar control logs and confirm the pod has
|
||||
`shareProcessNamespace: true`.
|
||||
The shared state directory should preserve `sandbox_gid` inheritance
|
||||
(`02775`). Sidecar SSH uses the Linux abstract socket
|
||||
`@openshell-sidecar-ssh`; the network sidecar verifies its peer PID before
|
||||
bridging gateway relay requests. No `ssh.sock` file should appear in the shared
|
||||
state directory.
|
||||
Inspect all three when sandbox registration or egress enforcement fails:
|
||||
Inspect the complete companion set when a proxy-pod sandbox remains Starting
|
||||
or loses readiness:
|
||||
|
||||
```bash
|
||||
kubectl -n openshell get configmap openshell-config -o jsonpath='{.data.gateway\.toml}' | grep -E '^\[openshell\.drivers\.kubernetes\]|^topology\s*='
|
||||
kubectl -n <sandbox-namespace> get pod <sandbox-pod> -o jsonpath='{range .spec.initContainers[*]}{.name}{" "}{.command}{"\n"}{end}'
|
||||
kubectl -n <sandbox-namespace> get pod <sandbox-pod> -o jsonpath='{range .spec.containers[*]}{.name}{" "}{.command}{"\n"}{end}'
|
||||
kubectl -n <sandbox-namespace> logs <sandbox-pod> -c openshell-network-init --tail=200
|
||||
kubectl -n <sandbox-namespace> logs <sandbox-pod> -c openshell-supervisor-network --tail=200
|
||||
kubectl -n <sandbox-namespace> logs <sandbox-pod> -c agent --tail=200
|
||||
kubectl -n <sandbox-namespace> get sandbox,pod,deployment,replicaset,service,secret,networkpolicy -l openshell.ai/sandbox-id=<sandbox-id>
|
||||
kubectl -n <sandbox-namespace> describe deployment -l openshell.ai/sandbox-id=<sandbox-id>
|
||||
kubectl -n <sandbox-namespace> logs deployment/<control-deployment> --tail=200
|
||||
kubectl -n <sandbox-namespace> get pod -l openshell.ai/sandbox-id=<sandbox-id>,openshell.ai/boundary-role=workload -o yaml
|
||||
kubectl -n <sandbox-namespace> get networkpolicy -l openshell.ai/sandbox-id=<sandbox-id> -o yaml
|
||||
```
|
||||
|
||||
Creation and recovery fail closed. A missing Secret leaves both pods inert; a
|
||||
missing or unobserved workload fence must prevent the driver from releasing the
|
||||
Sandbox; and readiness requires both Agent Sandbox readiness and an Available
|
||||
supervisor Deployment. Its exec readiness check succeeds only after the
|
||||
supervisor has attached, confirmed enforcement, started or resumed the
|
||||
workload, and registered the gateway access plane. Use both Pod logs for
|
||||
bootstrap errors. An `EPERM` during enforcement setup means the runtime blocked
|
||||
a required unprivileged seccomp, task-memory, or Landlock operation. Do not add
|
||||
capabilities, gateway egress, or credentials to the workload Pod as a
|
||||
workaround.
|
||||
|
||||
#### Corporate upstream proxy
|
||||
|
||||
When the deployment routes sandbox egress through a corporate HTTP forward
|
||||
@@ -617,26 +606,21 @@ helm -n openshell get values openshell | grep -A8 upstreamProxy
|
||||
```
|
||||
|
||||
Only `http://host:port` forward proxies are supported; `https://` proxy URLs and
|
||||
plain-HTTP egress are out of scope and rejected. Proxy credentials require
|
||||
`topology = "sidecar"` — combined topology shares the credential mount with the
|
||||
workload, so the gateway rejects credentials there. The credential Secret named
|
||||
by `proxy_auth_secret_name` must exist in the sandbox namespace with the key
|
||||
named by `proxy_auth_secret_key`, and Kubernetes will not create keys longer
|
||||
than 253 bytes or named `.`/`..`.
|
||||
plain-HTTP egress are out of scope and rejected. The credential Secret named by
|
||||
`proxy_auth_secret_name` must exist in the sandbox namespace with the key named
|
||||
by `proxy_auth_secret_key`, and Kubernetes will not create keys longer than 253
|
||||
bytes or named `.`/`..`.
|
||||
|
||||
The proxy arguments and credential mount are injected only into the container
|
||||
that runs network supervision (the `agent` container in combined topology, the
|
||||
`openshell-supervisor-network` sidecar in sidecar topology). The one-shot
|
||||
`openshell-network-init` container and the process `agent` container in sidecar
|
||||
topology must never receive them. The credential is projected read-only as the
|
||||
The proxy arguments and credential mount are injected only into the supervisor
|
||||
Pod. The sandbox Pod must never receive them. The credential is projected read-only as the
|
||||
`openshell-upstream-proxy-auth` volume at `/run/openshell/upstream-proxy-auth`
|
||||
and passed as `--upstream-proxy-auth-file`; it must never appear in env,
|
||||
annotations, or command arguments.
|
||||
|
||||
```bash
|
||||
kubectl -n <sandbox-namespace> get secret <proxy-auth-secret> -o jsonpath='{.data}' >/dev/null && echo "secret present"
|
||||
kubectl -n <sandbox-namespace> get pod <sandbox-pod> -o jsonpath='{range .spec.containers[*]}{.name}{" "}{.command}{"\n"}{end}' | grep -- '--upstream-'
|
||||
kubectl -n <sandbox-namespace> get pod <sandbox-pod> -o jsonpath='{range .spec.containers[*]}{.name}{": "}{range .volumeMounts[*]}{.name}{" "}{end}{"\n"}{end}' | grep upstream-proxy-auth
|
||||
kubectl -n <sandbox-namespace> get deployment <supervisor-deployment> -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{" "}{.command}{"\n"}{end}' | grep -- '--upstream-'
|
||||
kubectl -n <sandbox-namespace> get deployment <supervisor-deployment> -o jsonpath='{range .spec.template.spec.containers[*]}{.name}{": "}{range .volumeMounts[*]}{.name}{" "}{end}{"\n"}{end}' | grep upstream-proxy-auth
|
||||
kubectl -n <sandbox-namespace> get events --sort-by=.lastTimestamp | grep -Ei 'secret|MountVolume' | tail -n 20
|
||||
```
|
||||
|
||||
@@ -649,7 +633,7 @@ destination that should be direct is missing from `no_proxy`. Inspect the
|
||||
network supervisor logs for CONNECT and upstream-proxy decisions:
|
||||
|
||||
```bash
|
||||
kubectl -n <sandbox-namespace> logs <sandbox-pod> -c openshell-supervisor-network --tail=200 | grep -Ei 'upstream|connect|proxy'
|
||||
kubectl -n <sandbox-namespace> logs deployment/<supervisor-deployment> --tail=200 | grep -Ei 'upstream|connect|proxy'
|
||||
```
|
||||
|
||||
### Step 7: Check VM-Backed Gateways
|
||||
|
||||
@@ -71,46 +71,16 @@ description = "Run skaffold dev for deploy/helm/openshell (iterative deploy)"
|
||||
dir = "deploy/helm/openshell"
|
||||
run = "skaffold dev"
|
||||
|
||||
["helm:skaffold:dev:sidecar"]
|
||||
description = "Run skaffold dev with the Kubernetes supervisor sidecar topology"
|
||||
dir = "deploy/helm/openshell"
|
||||
run = "skaffold dev -p sidecar"
|
||||
|
||||
["helm:skaffold:dev:sidecar-mtls"]
|
||||
description = "Run skaffold dev with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled"
|
||||
dir = "deploy/helm/openshell"
|
||||
run = "skaffold dev -p sidecar-mtls"
|
||||
|
||||
["helm:skaffold:run"]
|
||||
description = "Run a one-shot Skaffold deploy and register its local plaintext gateway"
|
||||
dir = "deploy/helm/openshell"
|
||||
run = "skaffold run && ../../../tasks/scripts/helm-k3s-local.sh register"
|
||||
|
||||
["helm:skaffold:run:sidecar"]
|
||||
description = "Run a sidecar Skaffold deploy and register its local plaintext gateway"
|
||||
dir = "deploy/helm/openshell"
|
||||
run = "skaffold run -p sidecar && ../../../tasks/scripts/helm-k3s-local.sh register"
|
||||
|
||||
["helm:skaffold:run:sidecar-mtls"]
|
||||
description = "Run skaffold run with the Kubernetes supervisor sidecar topology and TLS/mTLS enabled"
|
||||
dir = "deploy/helm/openshell"
|
||||
run = "skaffold run -p sidecar-mtls"
|
||||
|
||||
["helm:skaffold:delete"]
|
||||
description = "Run skaffold delete for deploy/helm/openshell"
|
||||
dir = "deploy/helm/openshell"
|
||||
run = "skaffold delete"
|
||||
|
||||
["helm:skaffold:delete:sidecar"]
|
||||
description = "Run skaffold delete for the Kubernetes supervisor sidecar topology"
|
||||
dir = "deploy/helm/openshell"
|
||||
run = "skaffold delete -p sidecar"
|
||||
|
||||
["helm:skaffold:delete:sidecar-mtls"]
|
||||
description = "Run skaffold delete for the Kubernetes supervisor sidecar topology with TLS/mTLS enabled"
|
||||
dir = "deploy/helm/openshell"
|
||||
run = "skaffold delete -p sidecar-mtls"
|
||||
|
||||
["helm:skaffold:diagnose"]
|
||||
description = "Run skaffold diagnose for deploy/helm/openshell"
|
||||
dir = "deploy/helm/openshell"
|
||||
|
||||
+6
-6
@@ -190,9 +190,9 @@ run = [
|
||||
"OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" AGENT_SANDBOX_VERSION=v0.4.6 e2e/rust/e2e-kubernetes.sh",
|
||||
]
|
||||
|
||||
["e2e:kubernetes:sidecar"]
|
||||
description = "Run Kubernetes e2e with the supervisor sidecar topology overlay"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-sidecar.yaml" }
|
||||
["e2e:kubernetes:proxy-pod"]
|
||||
description = "Run Kubernetes e2e with the sandbox/supervisor proxy-pod topology"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
@@ -210,13 +210,13 @@ run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debu
|
||||
|
||||
["e2e:kubernetes:workspace-managed"]
|
||||
description = "Run Kubernetes e2e with managed workspace mode (auto-created per-workspace namespaces)"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" }
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml:deploy/helm/openshell/ci/values-workspace-managed.yaml", OPENSHELL_E2E_KUBE_IMAGE_PULL_SECRET = "e2e-regcred", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_managed", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-managed" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
["e2e:kubernetes:workspace-operator"]
|
||||
description = "Run Kubernetes e2e with operator workspace mode (pre-provisioned per-workspace namespaces)"
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" }
|
||||
env = { OPENSHELL_E2E_KUBE_EXTRA_VALUES = "deploy/helm/openshell/ci/values-proxy-pod.yaml:deploy/helm/openshell/ci/values-workspace-operator.yaml", OPENSHELL_E2E_KUBE_TEST = "workspace_namespace_operator", OPENSHELL_E2E_KUBERNETES_FEATURES = "e2e,e2e-kubernetes,e2e-kubernetes-workspace-operator" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
@@ -248,7 +248,7 @@ depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-vm.sh"
|
||||
|
||||
["e2e:kubernetes:external-driver"]
|
||||
description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver sidecar"
|
||||
description = "Run Kubernetes conformance with a driver-free gateway and external Kubernetes driver"
|
||||
env = { OPENSHELL_E2E_EXTERNAL_COMPUTE_DRIVER = "1", OPENSHELL_E2E_KUBE_BUILD_IMAGES = "1", OPENSHELL_E2E_KUBERNETES_FEATURES = "" }
|
||||
depends = ["e2e:conformance:build"]
|
||||
run = "OPENSHELL_CONFORMANCE_BIN=\"${OPENSHELL_CONFORMANCE_BIN:-$PWD/target/debug/openshell-conformance}\" e2e/rust/e2e-kubernetes.sh"
|
||||
|
||||
Reference in New Issue
Block a user