mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-19 05:50:41 +08:00
Philippe Martin
35e0a68e4a
* feat(kubernetes): support corporate proxy CA bundle The Kubernetes driver had no way to supply a CA bundle for the corporate egress proxy, so an `https://` proxy with a private CA, or a TLS-intercepting proxy, could not be used. Podman and VM already expose `proxy_ca_bundle`. Add `proxy_ca_bundle` to `[openshell.drivers.kubernetes]` as a path the gateway Pod reads. The gateway stages the PEM into the existing per-generation supervisor bootstrap Secret and passes `--upstream-proxy-ca-bundle` on the supervisor argv. That Secret is already immutable, owner-referenced and garbage-collected, and its volume mounts every key at /.openshell/supervisor with no items filter, so this needs no new object kind, volume, mount, or RBAC verb, and works in shared, managed and operator workspace modes. The bundle is deliberately read from the gateway's filesystem rather than referenced as an object in the sandbox namespace. It becomes a trust anchor for every upstream the sandbox reaches, so it must stay in the gateway's trust domain; the immutable staging Secret also keeps the anchor from changing underneath a running sandbox. Bound the staged bundle at 256 KiB. The shared reader's limit is exactly the apiserver's own Secret limit and the bootstrap Secret carries four other keys, so a bundle between the two would pass gateway startup and then fail every sandbox create with an opaque `data: Too long`. Delegate the URL, no_proxy, connect_by_hostname and ca_bundle rules to the shared validate_upstream_proxy_settings, keeping the Secret-specific credential block local: this driver accepts an explicit `proxy_auth_allow_insecure = false` without credentials, which the shared rules reject. This also fixes the acknowledgement being demanded for an `https://` proxy, where the credential travels inside the verified TLS session. Add auth_setting_label so the inline-credential diagnostic names the Secret keys instead of proxy_auth_file, which this driver rejects as an unknown key. Document that the bundle should carry only the CA that signs the proxy's certificate, or that an intercepting proxy re-signs upstream certificates with. Public roots already reach the sandbox through the supervisor image and its TLS stack, and the bundle is concatenated with that system store into a single boundary control frame, so a full merged trust bundle spends the frame budget on duplicated roots. The frame, not the apiserver Secret limit, is the tighter of the two ceilings in practice; raising the staging bound requires checking it. Closes #3443 Signed-off-by: Philippe Martin <phmartin@redhat.com> * fix(helm): quote proxy CA ConfigMap references Signed-off-by: Philippe Martin <phmartin@redhat.com> --------- Signed-off-by: Philippe Martin <phmartin@redhat.com>
307 lines
12 KiB
Smarty
307 lines
12 KiB
Smarty
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
{{/*
|
|
Gateway pod template shared by the StatefulSet and Deployment workload shapes.
|
|
*/}}
|
|
{{- define "openshell.gatewayPodTemplate" -}}
|
|
metadata:
|
|
annotations:
|
|
# Roll the gateway workload when the rendered gateway TOML changes - the
|
|
# gateway only reads /etc/openshell/gateway.toml at startup, so without
|
|
# this annotation a `helm upgrade` that only mutates the ConfigMap would
|
|
# leave pods running with stale config.
|
|
checksum/gateway-config: {{ include (print $.Template.BasePath "/gateway-config.yaml") . | sha256sum }}
|
|
{{- with .Values.podAnnotations }}
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
labels:
|
|
{{- include "openshell.labels" . | nindent 4 }}
|
|
{{- with .Values.podLabels }}
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
spec:
|
|
terminationGracePeriodSeconds: {{ .Values.podLifecycle.terminationGracePeriodSeconds }}
|
|
{{- with .Values.imagePullSecrets }}
|
|
imagePullSecrets:
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
serviceAccountName: {{ include "openshell.serviceAccountName" . }}
|
|
{{- if .Values.server.hostGatewayIP }}
|
|
hostAliases:
|
|
- ip: {{ .Values.server.hostGatewayIP | quote }}
|
|
hostnames:
|
|
- host.docker.internal
|
|
- host.openshell.internal
|
|
{{- end }}
|
|
{{- with .Values.podSecurityContext }}
|
|
securityContext:
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
containers:
|
|
- name: openshell-gateway
|
|
securityContext:
|
|
{{- toYaml .Values.securityContext | nindent 8 }}
|
|
image: {{ include "openshell.image" . | quote }}
|
|
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
|
args:
|
|
- --config
|
|
- /etc/openshell/gateway.toml
|
|
{{- if not .Values.server.externalDbSecret }}
|
|
- --db-url
|
|
- {{ .Values.server.dbUrl | quote }}
|
|
{{- end }}
|
|
env:
|
|
- name: OPENSHELL_REPLICA_ID
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.name
|
|
- name: OPENSHELL_POD_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.name
|
|
- name: OPENSHELL_POD_NAMESPACE
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: metadata.namespace
|
|
{{- if eq (include "openshell.workloadKind" .) "deployment" }}
|
|
- name: OPENSHELL_POD_IP
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: status.podIP
|
|
- name: OPENSHELL_PEER_ENDPOINT
|
|
value: {{ printf "%s://$(OPENSHELL_POD_IP):%d" (ternary "http" "https" (default false .Values.server.disableTls)) (int .Values.service.port) | quote }}
|
|
{{- end }}
|
|
- name: OPENSHELL_SERVICE_ACCOUNT_NAME
|
|
value: {{ include "openshell.serviceAccountName" . | quote }}
|
|
- name: OPENSHELL_PEER_SERVICE_NAME
|
|
value: {{ include "openshell.peerServiceName" . | quote }}
|
|
- name: OPENSHELL_PEER_TOKEN_AUDIENCE
|
|
value: "openshell-gateway-peer"
|
|
- name: OPENSHELL_PEER_SERVICE_ACCOUNT_TOKEN_FILE
|
|
value: /var/run/secrets/openshell-peer/token
|
|
- name: OPENSHELL_PEER_POD_LABELS
|
|
value: {{ printf "app.kubernetes.io/name=%s,app.kubernetes.io/instance=%s" (include "openshell.name" .) .Release.Name | quote }}
|
|
{{- if not .Values.server.disableTls }}
|
|
- name: OPENSHELL_PEER_TLS_SERVER_NAME
|
|
value: {{ printf "%s.%s.svc.cluster.local" (include "openshell.fullname" .) .Release.Namespace | quote }}
|
|
{{- if or .Values.pkiInitJob.enabled .Values.certManager.enabled }}
|
|
- name: OPENSHELL_PEER_TLS_CA_FILE
|
|
value: /etc/openshell-tls/server/ca.crt
|
|
{{- end }}
|
|
{{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }}
|
|
- name: OPENSHELL_PEER_TLS_CERT_FILE
|
|
value: /etc/openshell-tls/peer-client/tls.crt
|
|
- name: OPENSHELL_PEER_TLS_KEY_FILE
|
|
value: /etc/openshell-tls/peer-client/tls.key
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if not (or .Values.server.credentialDrivers.kubernetesSecrets.enabled .Values.server.credentialDrivers.vault.enabled) }}
|
|
- name: {{ include "openshell.credentialStorageKeyEncryptionKeyEnvName" . }}
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ include "openshell.credentialStorageKeyEncryptionKeySecretName" . }}
|
|
key: {{ include "openshell.credentialStorageKeyEncryptionKeySecretKey" . }}
|
|
{{- end }}
|
|
{{- if .Values.server.externalDbSecret }}
|
|
- name: OPENSHELL_DB_URL
|
|
valueFrom:
|
|
secretKeyRef:
|
|
name: {{ .Values.server.externalDbSecret }}
|
|
key: uri
|
|
{{- end }}
|
|
# Most gateway settings live in the ConfigMap-backed TOML file
|
|
# mounted at /etc/openshell/gateway.toml. Secret-bearing settings use
|
|
# env vars that the TOML references by name. Some process-level
|
|
# settings consumed by libraries outside gateway code also remain here.
|
|
{{- if and .Values.server.oidc.issuer .Values.server.oidc.caConfigMapName }}
|
|
# OIDC issuer custom-CA: rustls/reqwest read SSL_CERT_FILE for
|
|
# outbound TLS verification. This is a process-level env var
|
|
# consumed by the TLS stack itself, not by gateway code, so it
|
|
# cannot be represented in the gateway TOML schema.
|
|
- name: SSL_CERT_FILE
|
|
value: /etc/openshell-tls/oidc-ca/ca.crt
|
|
{{- end }}
|
|
- name: OPENSHELL_TELEMETRY_ENABLED
|
|
value: {{ .Values.server.telemetryEnabled | quote }}
|
|
{{- if .Values.server.providerTokenGrants.spiffe.enabled }}
|
|
- name: OPENSHELL_GATEWAY_SPIFFE_WORKLOAD_API_SOCKET
|
|
value: {{ .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }}
|
|
{{- end }}
|
|
volumeMounts:
|
|
{{- if eq (include "openshell.workloadKind" .) "statefulset" }}
|
|
- name: openshell-data
|
|
mountPath: /var/openshell
|
|
{{- end }}
|
|
# ConfigMap directory mounts expose keys through atomic-writer symlinks,
|
|
# while the gateway intentionally rejects symlinked configuration.
|
|
# The checksum annotation above rolls pods when this subPath changes.
|
|
- name: gateway-config
|
|
mountPath: /etc/openshell/gateway.toml
|
|
subPath: gateway.toml
|
|
readOnly: true
|
|
- name: sandbox-jwt
|
|
mountPath: /etc/openshell-jwt
|
|
readOnly: true
|
|
- name: gateway-peer-token
|
|
mountPath: /var/run/secrets/openshell-peer
|
|
readOnly: true
|
|
{{- if not .Values.server.disableTls }}
|
|
- name: tls-cert
|
|
mountPath: /etc/openshell-tls/server
|
|
readOnly: true
|
|
{{- if .Values.certManager.serverIssuerRef.name }}
|
|
- name: tls-external-cert
|
|
mountPath: /etc/openshell-tls/server-external
|
|
readOnly: true
|
|
{{- end }}
|
|
{{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }}
|
|
- name: peer-client-tls
|
|
mountPath: /etc/openshell-tls/peer-client
|
|
readOnly: true
|
|
- name: tls-client-ca
|
|
mountPath: /etc/openshell-tls/client-ca
|
|
readOnly: true
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if and .Values.server.oidc.issuer .Values.server.oidc.caConfigMapName }}
|
|
- name: oidc-ca
|
|
mountPath: /etc/openshell-tls/oidc-ca
|
|
readOnly: true
|
|
{{- end }}
|
|
{{- if and .Values.server.credentialDrivers.vault.enabled .Values.server.credentialDrivers.vault.caConfigMapName }}
|
|
- name: vault-ca
|
|
mountPath: /etc/openshell-tls/vault-ca
|
|
readOnly: true
|
|
{{- end }}
|
|
{{- if .Values.upstreamProxy.caBundle.configMapName }}
|
|
- name: upstream-proxy-ca
|
|
mountPath: /etc/openshell-tls/proxy-ca
|
|
readOnly: true
|
|
{{- end }}
|
|
{{- if .Values.server.providerTokenGrants.spiffe.enabled }}
|
|
- name: spiffe-workload-api
|
|
mountPath: {{ dir .Values.server.providerTokenGrants.spiffe.workloadApiSocketPath | quote }}
|
|
readOnly: true
|
|
{{- end }}
|
|
ports:
|
|
- name: grpc
|
|
containerPort: {{ .Values.service.port }}
|
|
protocol: TCP
|
|
- name: health
|
|
containerPort: {{ .Values.service.healthPort }}
|
|
protocol: TCP
|
|
{{- if .Values.service.metricsPort }}
|
|
- name: metrics
|
|
containerPort: {{ .Values.service.metricsPort }}
|
|
protocol: TCP
|
|
{{- end }}
|
|
startupProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: health
|
|
periodSeconds: {{ .Values.probes.startup.periodSeconds }}
|
|
timeoutSeconds: {{ .Values.probes.startup.timeoutSeconds }}
|
|
failureThreshold: {{ .Values.probes.startup.failureThreshold }}
|
|
livenessProbe:
|
|
httpGet:
|
|
path: /healthz
|
|
port: health
|
|
initialDelaySeconds: {{ .Values.probes.liveness.initialDelaySeconds }}
|
|
periodSeconds: {{ .Values.probes.liveness.periodSeconds }}
|
|
timeoutSeconds: {{ .Values.probes.liveness.timeoutSeconds }}
|
|
failureThreshold: {{ .Values.probes.liveness.failureThreshold }}
|
|
readinessProbe:
|
|
httpGet:
|
|
path: /readyz
|
|
port: health
|
|
initialDelaySeconds: {{ .Values.probes.readiness.initialDelaySeconds }}
|
|
periodSeconds: {{ .Values.probes.readiness.periodSeconds }}
|
|
timeoutSeconds: {{ .Values.probes.readiness.timeoutSeconds }}
|
|
failureThreshold: {{ .Values.probes.readiness.failureThreshold }}
|
|
resources:
|
|
{{- toYaml .Values.resources | nindent 8 }}
|
|
volumes:
|
|
- name: gateway-config
|
|
configMap:
|
|
name: {{ include "openshell.fullname" . }}-config
|
|
- name: sandbox-jwt
|
|
secret:
|
|
secretName: {{ include "openshell.sandboxJwtSecretName" . }}
|
|
defaultMode: {{ .Values.server.sandboxJwt.secretDefaultMode | default 0400 }}
|
|
- name: gateway-peer-token
|
|
projected:
|
|
defaultMode: 0400
|
|
sources:
|
|
- serviceAccountToken:
|
|
path: token
|
|
audience: openshell-gateway-peer
|
|
expirationSeconds: 3600
|
|
{{- if not .Values.server.disableTls }}
|
|
- name: tls-cert
|
|
secret:
|
|
secretName: {{ .Values.server.tls.certSecretName }}
|
|
{{- if .Values.certManager.serverIssuerRef.name }}
|
|
- name: tls-external-cert
|
|
secret:
|
|
secretName: {{ include "openshell.fullname" . }}-server-external-tls
|
|
{{- end }}
|
|
{{- if eq (include "openshell.gatewayClientCaEnabled" .) "true" }}
|
|
- name: peer-client-tls
|
|
secret:
|
|
secretName: {{ .Values.server.tls.clientTlsSecretName }}
|
|
- name: tls-client-ca
|
|
secret:
|
|
{{- if or (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) }}
|
|
secretName: {{ .Values.server.tls.certSecretName }}
|
|
items:
|
|
- key: ca.crt
|
|
path: ca.crt
|
|
{{- else }}
|
|
secretName: {{ .Values.server.tls.clientCaSecretName }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- if and .Values.server.oidc.issuer .Values.server.oidc.caConfigMapName }}
|
|
- name: oidc-ca
|
|
configMap:
|
|
name: {{ .Values.server.oidc.caConfigMapName }}
|
|
{{- end }}
|
|
{{- if and .Values.server.credentialDrivers.vault.enabled .Values.server.credentialDrivers.vault.caConfigMapName }}
|
|
- name: vault-ca
|
|
configMap:
|
|
name: {{ .Values.server.credentialDrivers.vault.caConfigMapName }}
|
|
items:
|
|
- key: ca.crt
|
|
path: ca.crt
|
|
{{- end }}
|
|
{{- if .Values.upstreamProxy.caBundle.configMapName }}
|
|
- name: upstream-proxy-ca
|
|
configMap:
|
|
name: {{ .Values.upstreamProxy.caBundle.configMapName | quote }}
|
|
items:
|
|
# The mounted filename stays fixed so the rendered proxy_ca_bundle
|
|
# path does not depend on the operator's ConfigMap key.
|
|
- key: {{ .Values.upstreamProxy.caBundle.key | default "ca.crt" | quote }}
|
|
path: ca.crt
|
|
{{- end }}
|
|
{{- if .Values.server.providerTokenGrants.spiffe.enabled }}
|
|
- name: spiffe-workload-api
|
|
csi:
|
|
driver: csi.spiffe.io
|
|
readOnly: true
|
|
{{- end }}
|
|
{{- with .Values.nodeSelector }}
|
|
nodeSelector:
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
{{- with .Values.affinity }}
|
|
affinity:
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
{{- with .Values.tolerations }}
|
|
tolerations:
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
{{- end }}
|