mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-31 07:44:57 +08:00
feat(kubernetes): deploy shared supervisor mode (#4695)
* feat(kubernetes): bootstrap experimental gateway supervisor topology Signed-off-by: Drew Newberry <anewberry@nvidia.com> * docs: move the experimental supervisor topology under an Experimental section Signed-off-by: Drew Newberry <anewberry@nvidia.com> * feat(supervisor): report experimental bootstrap health over TCP experimental-bootstrap now takes --health-port instead of a Unix health socket. It binds the dedicated supervisor's wildcard readiness listener, reachable at the Pod IP, and accepts and closes connections while the process runs. The chart probes the named container port health (5502) with native kubelet tcpSocket checks and drops the health-only emptyDir. No filesystem state remains to clean up after an unclean exit. Dedicated supervisors keep --health-socket-path and health --socket. Signed-off-by: Drew Newberry <anewberry@nvidia.com> * docs: defer public experimental supervisor docs Remove the experimental supervisor topology additions from the published gateway configuration page and the public debug-openshell-cluster skill. Public documentation will be added once the experimental stack settles. Signed-off-by: Drew Newberry <anewberry@nvidia.com> * refactor(supervisor): replace bootstrap with shared host stub Signed-off-by: Drew Newberry <anewberry@nvidia.com> * refactor(supervisor): name shared supervisor subcommand shared Signed-off-by: Drew Newberry <anewberry@nvidia.com> * refactor(supervisor): select shared mode with explicit endpoint flags Signed-off-by: Drew Newberry <anewberry@nvidia.com> * refactor(supervisor): name the shared host module after its mode Rename the shared-mode host module from experimental to shared to match the --mode=shared flag. No behavior changes. Signed-off-by: Drew Newberry <anewberry@nvidia.com> * refactor(kubernetes): drop the experimental prefix from the supervisor topology Rename the Helm template to supervisor.yaml, its Kubernetes names and labels to the -supervisor suffix and supervisor component, and the driver setting to supervisor_topology. The setting remains opt-in and workspace remains reserved. Signed-off-by: Drew Newberry <anewberry@nvidia.com> --------- Signed-off-by: Drew Newberry <anewberry@nvidia.com>
This commit is contained in:
@@ -57,6 +57,27 @@ Override with env vars before running `helm:k3s:create`:
|
||||
`mcr.microsoft.com/dotnet/aspire-dashboard:latest`)
|
||||
- `HELM_K3S_COLLECTOR_HEALTH_TIMEOUT` (default: `120` seconds)
|
||||
|
||||
### Shared supervisor plumbing
|
||||
|
||||
Set `gatewayConfig.openshell.drivers.kubernetes.supervisor_topology`
|
||||
using a values file (the table key is literally `openshell.drivers.kubernetes`):
|
||||
|
||||
```yaml
|
||||
gatewayConfig:
|
||||
openshell.drivers.kubernetes:
|
||||
supervisor_topology: gateway
|
||||
```
|
||||
|
||||
The gateway chart creates a separate, single-replica shared-host placeholder Deployment
|
||||
in the release namespace, regardless of shared/managed/operator workspace mode.
|
||||
Use a supervisor image built from this branch: `--mode=shared` is new.
|
||||
The chart sets `--gateway-endpoint` from the gateway's gRPC endpoint and
|
||||
`--address` to `0.0.0.0:5501`. Check its rollout and TCP health probes on this
|
||||
listener (container port `supervisor`, 5501). The outbound gateway connection and
|
||||
session RPCs are not implemented yet; sandboxes
|
||||
still use dedicated supervisors. `workspace` is reserved and rejected. Omit the
|
||||
field to disable it.
|
||||
|
||||
### 2. Deploy OpenShell
|
||||
|
||||
**Iterative dev** (rebuilds on file changes, recommended during active development):
|
||||
|
||||
@@ -194,9 +194,20 @@ where
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// Scope of a shared supervisor deployment.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SupervisorTopology {
|
||||
Gateway,
|
||||
Workspace,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub struct KubernetesComputeConfig {
|
||||
/// Deploy an idle shared supervisor through Helm. Does not route sessions.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub supervisor_topology: Option<SupervisorTopology>,
|
||||
/// Permit caller-supplied driver JSON. Does not waive resource admission.
|
||||
pub allow_driver_config: bool,
|
||||
/// Operator-owned external attachment approval policy.
|
||||
@@ -389,6 +400,7 @@ pub const ANNOTATION_SCC_SUPPLEMENTAL_GROUPS: &str = "openshift.io/sa.scc.supple
|
||||
impl Default for KubernetesComputeConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
supervisor_topology: None,
|
||||
workspace_mode: WorkspaceMode::default(),
|
||||
allow_driver_config: false,
|
||||
resource_admission:
|
||||
@@ -442,6 +454,9 @@ impl Default for KubernetesComputeConfig {
|
||||
impl KubernetesComputeConfig {
|
||||
/// Validate startup configuration without connecting to Kubernetes.
|
||||
pub fn validate_configuration(&self) -> Result<(), String> {
|
||||
if self.supervisor_topology == Some(SupervisorTopology::Workspace) {
|
||||
return Err("supervisor_topology=workspace is not implemented".into());
|
||||
}
|
||||
self.validate_workspace_mode()?;
|
||||
self.validate_provider_spiffe_workload_api_socket_path()?;
|
||||
self.validate_sandbox_identity_config()?;
|
||||
|
||||
@@ -13,7 +13,7 @@ mod sandbox_runtime;
|
||||
pub use config::{
|
||||
DEFAULT_GATEWAY_ID, DEFAULT_SANDBOX_SERVICE_ACCOUNT_NAME, DEFAULT_WORKSPACE_STORAGE_SIZE,
|
||||
KubernetesComputeConfig, KubernetesControlConnection, KubernetesImagePullPolicy,
|
||||
KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, WorkspaceMode,
|
||||
KubernetesSandboxRuntimeConfig, ManagedSshIngressConfig, SupervisorTopology, WorkspaceMode,
|
||||
managed_namespace_prefix,
|
||||
};
|
||||
pub use driver::{KubernetesComputeDriver, KubernetesDriverError};
|
||||
|
||||
@@ -260,6 +260,7 @@ async fn main() -> Result<()> {
|
||||
let (shutdown_tx, shutdown_rx) = tokio::sync::watch::channel(false);
|
||||
let driver = KubernetesComputeDriver::new(
|
||||
KubernetesComputeConfig {
|
||||
supervisor_topology: None,
|
||||
allow_driver_config: args.admission_config_json.allow_driver_config,
|
||||
resource_admission: args.admission_config_json.resource_admission.clone(),
|
||||
workspace_mode: args.workspace_mode,
|
||||
|
||||
@@ -60,3 +60,16 @@ belong to the sandbox or isolation backend. Signals use `BoundarySignal`.
|
||||
|
||||
These portable control-plane foundations do not qualify a platform isolation
|
||||
backend or enable a Windows workload runtime.
|
||||
|
||||
## Shared host
|
||||
|
||||
`openshell-supervisor --mode=shared --gateway-endpoint https://gateway:8080 --address 0.0.0.0:5501`
|
||||
starts a shared-host placeholder for the Kubernetes gateway supervisor topology.
|
||||
The listener at `--address` accepts and closes connections, including kubelet
|
||||
`tcpSocket` probes, and reports process health only. `--gateway-endpoint` selects
|
||||
the gateway the supervisor will dial for session control. The outbound connection,
|
||||
control RPCs, sandbox sessions, gateway authentication, and tenant routing are not
|
||||
implemented yet. Omitting `--mode` keeps the existing dedicated behavior.
|
||||
`supervisor_topology = "gateway"` deploys this placeholder; sandboxes
|
||||
continue using dedicated supervisors. SIGTERM and SIGINT close the listener.
|
||||
Shared mode is Unix-only.
|
||||
|
||||
@@ -18,11 +18,21 @@ use tracing_subscriber::filter::LevelFilter;
|
||||
use tracing_subscriber::{Layer as _, layer::SubscriberExt as _, util::SubscriberInitExt as _};
|
||||
|
||||
mod logging;
|
||||
mod shared;
|
||||
|
||||
const DEBUG_RPC_SUBCOMMAND: &str = "debug-rpc";
|
||||
const HEALTH_SUBCOMMAND: &str = "health";
|
||||
const SERVICE_NAME: &str = "openshell-supervisor";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)]
|
||||
enum SupervisorMode {
|
||||
/// Supervise a single sandbox generation.
|
||||
#[default]
|
||||
Dedicated,
|
||||
/// Host sandbox supervision sessions in a shared process.
|
||||
Shared,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, ValueEnum)]
|
||||
enum SupervisorRole {
|
||||
/// Attach an Isolation Backend and supervise one sandbox generation.
|
||||
@@ -46,6 +56,18 @@ struct HealthArgs {
|
||||
#[command(about = "OpenShell policy and workload supervisor")]
|
||||
#[allow(clippy::struct_excessive_bools)]
|
||||
struct Args {
|
||||
/// Supervisor deployment mode.
|
||||
#[arg(long, value_enum, default_value_t)]
|
||||
mode: SupervisorMode,
|
||||
|
||||
/// Gateway endpoint for the shared supervisor's outbound connection.
|
||||
#[arg(long, required_if_eq("mode", "shared"))]
|
||||
gateway_endpoint: Option<String>,
|
||||
|
||||
/// Shared supervisor listen address, also used for TCP health probes.
|
||||
#[arg(long, required_if_eq("mode", "shared"))]
|
||||
address: Option<std::net::SocketAddr>,
|
||||
|
||||
/// Supervisor responsibility to run.
|
||||
#[arg(long, value_enum, default_value_t)]
|
||||
role: SupervisorRole,
|
||||
@@ -366,6 +388,21 @@ fn main() -> Result<()> {
|
||||
}
|
||||
|
||||
let args = Args::parse();
|
||||
if args.mode == SupervisorMode::Shared {
|
||||
let address = args
|
||||
.address
|
||||
.ok_or_else(|| miette::miette!("--address is required for --mode=shared"))?;
|
||||
let gateway_endpoint = args
|
||||
.gateway_endpoint
|
||||
.as_deref()
|
||||
.ok_or_else(|| miette::miette!("--gateway-endpoint is required for --mode=shared"))?;
|
||||
return shared::run(address, gateway_endpoint);
|
||||
}
|
||||
if args.gateway_endpoint.is_some() || args.address.is_some() {
|
||||
return Err(miette::miette!(
|
||||
"--gateway-endpoint and --address require --mode=shared"
|
||||
));
|
||||
}
|
||||
validate_role_arguments(&args)?;
|
||||
arm_parent_liveness(args.parent_liveness_stdin)?;
|
||||
validate_main_exit_marker(args.main_exit_marker.as_deref())?;
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! Shared-host placeholder: no sandbox admission, credentials, or session RPCs.
|
||||
|
||||
#[cfg(unix)]
|
||||
use miette::IntoDiagnostic;
|
||||
use miette::Result;
|
||||
use std::net::SocketAddr;
|
||||
|
||||
// The gateway endpoint is reserved for the shared supervisor's outbound session.
|
||||
// This plumbing stage only starts the local listener.
|
||||
pub fn run(address: SocketAddr, _gateway_endpoint: &str) -> Result<()> {
|
||||
run_platform(address)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn run_platform(address: SocketAddr) -> Result<()> {
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.into_diagnostic()?;
|
||||
runtime.block_on(async {
|
||||
// Install shutdown handlers before publishing health.
|
||||
let mut terminate =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
|
||||
.into_diagnostic()?;
|
||||
let mut interrupt =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::interrupt())
|
||||
.into_diagnostic()?;
|
||||
let listener = tokio::net::TcpListener::bind(address)
|
||||
.await
|
||||
.into_diagnostic()?;
|
||||
eprintln!(
|
||||
"Shared supervisor started; gateway connection and sandbox sessions are not implemented"
|
||||
);
|
||||
loop {
|
||||
tokio::select! {
|
||||
// This placeholder only accepts and closes connections, including
|
||||
// health probes. Session control will use an outbound gateway stream.
|
||||
result = listener.accept() => {
|
||||
if let Err(error) = result {
|
||||
eprintln!("shared supervisor listener accept failed: {error}");
|
||||
tokio::time::sleep(std::time::Duration::from_millis(100)).await;
|
||||
}
|
||||
}
|
||||
_ = terminate.recv() => break,
|
||||
_ = interrupt.recv() => break,
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn run_platform(_address: SocketAddr) -> Result<()> {
|
||||
Err(miette::miette!("--mode=shared requires Unix"))
|
||||
}
|
||||
@@ -568,6 +568,20 @@ like user-supplied schema-v2 fields during compatibility resolution. */}}
|
||||
Validate chart values that Helm would otherwise accept silently.
|
||||
*/}}
|
||||
{{- define "openshell.validateValues" -}}
|
||||
{{- $kubernetes := include "openshell.effectiveKubernetesConfig" . | fromYaml -}}
|
||||
{{- if hasKey $kubernetes "supervisor_topology" -}}
|
||||
{{- $topology := get $kubernetes "supervisor_topology" -}}
|
||||
{{- if not (has $topology (list "gateway" "workspace")) -}}
|
||||
{{- fail "supervisor_topology must be gateway or workspace; omit it to disable the shared supervisor" -}}
|
||||
{{- end -}}
|
||||
{{- if eq $topology "workspace" -}}
|
||||
{{- fail "supervisor_topology=workspace is not implemented" -}}
|
||||
{{- end -}}
|
||||
{{- $gateway := get (.Values.gatewayConfig | default dict) "openshell.gateway" | default dict -}}
|
||||
{{- if ne (get $gateway "compute_driver" | default "kubernetes") "kubernetes" -}}
|
||||
{{- fail "supervisor_topology requires the kubernetes compute driver" -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
{{- $workloadKind := include "openshell.workloadKind" . -}}
|
||||
{{- $workload := .Values.workload | default dict -}}
|
||||
{{- $maxReplicas := int (include "openshell.maxReplicas" .) -}}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
{{- include "openshell.validateValues" . }}
|
||||
{{- $kubernetes := include "openshell.effectiveKubernetesConfig" . | fromYaml -}}
|
||||
{{- $imagePullSecrets := get $kubernetes "image_pull_secrets" | default list -}}
|
||||
{{- range .Values.imagePullSecrets -}}
|
||||
{{- $imagePullSecrets = append $imagePullSecrets .name -}}
|
||||
{{- end -}}
|
||||
{{- if eq (get $kubernetes "supervisor_topology" | default "") "gateway" }}
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ include "openshell.fullname" . }}-supervisor
|
||||
namespace: {{ .Release.Namespace }}
|
||||
labels:
|
||||
{{- include "openshell.labels" . | nindent 4 }}
|
||||
app.kubernetes.io/component: supervisor
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: openshell-supervisor
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: supervisor
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: openshell-supervisor
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
app.kubernetes.io/component: supervisor
|
||||
spec:
|
||||
automountServiceAccountToken: false
|
||||
terminationGracePeriodSeconds: 30
|
||||
{{- with ($imagePullSecrets | uniq) }}
|
||||
imagePullSecrets:
|
||||
{{- range . }}
|
||||
- name: {{ . | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
securityContext:
|
||||
runAsNonRoot: true
|
||||
runAsUser: 65534
|
||||
runAsGroup: 65534
|
||||
fsGroup: 65534
|
||||
seccompProfile:
|
||||
type: RuntimeDefault
|
||||
containers:
|
||||
- name: supervisor
|
||||
image: {{ get $kubernetes "supervisor_image" | quote }}
|
||||
imagePullPolicy: {{ get (dict "always" "Always" "if_not_present" "IfNotPresent" "never" "Never") (get $kubernetes "supervisor_image_pull_policy" | default "if_not_present") | quote }}
|
||||
args:
|
||||
- --mode=shared
|
||||
- --gateway-endpoint
|
||||
- {{ include "openshell.grpcEndpoint" . | quote }}
|
||||
- --address
|
||||
- 0.0.0.0:5501
|
||||
ports:
|
||||
- name: supervisor
|
||||
containerPort: 5501
|
||||
protocol: TCP
|
||||
securityContext:
|
||||
allowPrivilegeEscalation: false
|
||||
readOnlyRootFilesystem: true
|
||||
capabilities:
|
||||
drop: [ALL]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
memory: 128Mi
|
||||
readinessProbe:
|
||||
tcpSocket:
|
||||
port: supervisor
|
||||
livenessProbe:
|
||||
tcpSocket:
|
||||
port: supervisor
|
||||
{{- end }}
|
||||
@@ -0,0 +1,105 @@
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
suite: supervisor deployment
|
||||
templates:
|
||||
- templates/supervisor.yaml
|
||||
- templates/gateway-config.yaml
|
||||
release:
|
||||
name: openshell
|
||||
namespace: gateway
|
||||
tests:
|
||||
- it: adds no supervisor by default
|
||||
template: templates/supervisor.yaml
|
||||
asserts:
|
||||
- hasDocuments:
|
||||
count: 0
|
||||
- it: deploys one isolated idle process in the gateway namespace
|
||||
template: templates/supervisor.yaml
|
||||
set:
|
||||
gatewayConfig:
|
||||
openshell.drivers.kubernetes:
|
||||
supervisor_topology: gateway
|
||||
supervisor_image: example.com/supervisor:shared
|
||||
asserts:
|
||||
- equal:
|
||||
path: metadata.namespace
|
||||
value: gateway
|
||||
- equal:
|
||||
path: spec.replicas
|
||||
value: 1
|
||||
- equal:
|
||||
path: spec.template.metadata.labels["app.kubernetes.io/name"]
|
||||
value: openshell-supervisor
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].image
|
||||
value: example.com/supervisor:shared
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].args
|
||||
value:
|
||||
- --mode=shared
|
||||
- --gateway-endpoint
|
||||
- https://openshell.gateway.svc.cluster.local:8080
|
||||
- --address
|
||||
- 0.0.0.0:5501
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].ports
|
||||
value:
|
||||
- name: supervisor
|
||||
containerPort: 5501
|
||||
protocol: TCP
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].readinessProbe
|
||||
value:
|
||||
tcpSocket:
|
||||
port: supervisor
|
||||
- equal:
|
||||
path: spec.template.spec.containers[0].livenessProbe
|
||||
value:
|
||||
tcpSocket:
|
||||
port: supervisor
|
||||
- notExists:
|
||||
path: spec.template.spec.containers[0].volumeMounts
|
||||
- notExists:
|
||||
path: spec.template.spec.volumes
|
||||
- equal:
|
||||
path: spec.template.spec.automountServiceAccountToken
|
||||
value: false
|
||||
- it: renders the same setting into gateway TOML
|
||||
template: templates/gateway-config.yaml
|
||||
set:
|
||||
gatewayConfig:
|
||||
openshell.drivers.kubernetes:
|
||||
supervisor_topology: gateway
|
||||
asserts:
|
||||
- matchRegex:
|
||||
path: data["gateway.toml"]
|
||||
pattern: 'supervisor_topology = "gateway"'
|
||||
- it: rejects workspace until implemented
|
||||
template: templates/supervisor.yaml
|
||||
set:
|
||||
gatewayConfig:
|
||||
openshell.drivers.kubernetes:
|
||||
supervisor_topology: workspace
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: supervisor_topology=workspace is not implemented
|
||||
- it: rejects invalid topology
|
||||
template: templates/supervisor.yaml
|
||||
set:
|
||||
gatewayConfig:
|
||||
openshell.drivers.kubernetes:
|
||||
supervisor_topology: invalid
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: supervisor_topology must be gateway or workspace; omit it to disable the shared supervisor
|
||||
- it: rejects enabled topology for another compute driver
|
||||
template: templates/supervisor.yaml
|
||||
set:
|
||||
gatewayConfig:
|
||||
openshell.gateway:
|
||||
compute_driver: docker
|
||||
openshell.drivers.kubernetes:
|
||||
supervisor_topology: gateway
|
||||
asserts:
|
||||
- failedTemplate:
|
||||
errorMessage: supervisor_topology requires the kubernetes compute driver
|
||||
Reference in New Issue
Block a user