mirror of
https://github.com/NVIDIA/OpenShell.git
synced 2026-10-20 06:00:06 +08:00
* feat(kubernetes): support HA gateway rebalancing Signed-off-by: Drew Newberry <anewberry@nvidia.com> * perf(server): cache peer connections, tokens, and owner lookups Every forwarded relay rebuilt its setup from scratch: an owner lookup, a blocking read of the peer token, a TLS connect to the owning replica, and a TokenReview plus Pod GET on the receiving side. Sandbox service routing does this per HTTP request, so the apiserver calls scaled with traffic. Cache all of it on ServerState: - peer channels pooled per endpoint, so relays multiplex over one connection instead of redialing - peer tokens keyed by SHA-256, expiring at min(ttl, token exp) so a hit cannot accept an expired token - owner records for 3s against a 45s ownership TTL, still freshness checked before use Entries are evicted when a relay fails. Also raise HTTP/2 max_concurrent_streams to 1024, since pooling funnels every relay between two replicas onto one connection and hyper's default of 200 sits below the 256 pending-relay budget. Signed-off-by: divesh <dgude@nvidia.com> * perf(server): pool upstream connections for sandbox services Each HTTP request to a sandbox service opened its own supervisor relay, paying a new TCP connection and HTTP/1 handshake every time. Worse, it counted against the 32 in-flight relay cap, so a service handling more than 32 concurrent requests failed outright. Pool idle upstreams per endpoint and port, up to 8 each for 15s. Reuse is safe because the pool only returns a connection hyper reports as ready, and HTTP/1 cannot start a request until the previous body has drained. Upgrades are never pooled since they take the connection over, and a failed send evicts that endpoint. Pruning is bounded per key, with the full sweep limited to once per 30s. Signed-off-by: divesh <dgude@nvidia.com> * fix(server): address HA gateway review findings (#3449) - Let a gateway own supervisor sessions without a peer endpoint. Requiring one whenever the store is PostgreSQL broke every single-instance PostgreSQL deployment, because no sandbox supervisor could connect. A cross-replica request to an owner that advertises no endpoint now fails immediately naming the cause, instead of retrying until the wait timeout. - Close a supervisor session on heartbeat only when another replica owns it, or after renewals fail for the ownership TTL. A database error no longer drops every session heartbeating during an outage. - Clamp owner record ages at zero so a skewed or corrupt stored timestamp cannot produce a negative age. - Bound the cross-object advisory lock with a lock timeout, so a stuck holder fails instead of blocking every mutation in the fleet. - Refuse to start when a peer endpoint is configured on a multi-replica backend but peer authentication is unavailable, and warn when a multi-replica backend has no peer endpoint at all. - Reject a plaintext peer endpoint when the gateway serves TLS. - Skip the sandbox watch poller on single-replica backends, where the local update bus already sees every write. - Rate-limit the peer owner cache sweep so an insert no longer scans the whole map under the lock. - Retry GET and HEAD on a pooled upstream the sandbox closed, instead of returning 502, and drop an emptied endpoint from the pool right away. - Document the gateway peer environment variables and the post-rollout ownership skew operators should expect. Signed-off-by: divesh <dgude@nvidia.com> * fix(server): harden HA supervisor ownership Signed-off-by: Drew Newberry <anewberry@nvidia.com> --------- Signed-off-by: Drew Newberry <anewberry@nvidia.com> Signed-off-by: divesh <dgude@nvidia.com> Co-authored-by: Drew Newberry <anewberry@nvidia.com> Co-authored-by: divesh <dgude@nvidia.com> Co-authored-by: Divesh Chowdary <47188680+FrostGod@users.noreply.github.com>
330 lines
13 KiB
Smarty
330 lines
13 KiB
Smarty
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
{{/*
|
|
Expand the name of the chart.
|
|
*/}}
|
|
{{- define "openshell.name" -}}
|
|
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Create a default fully qualified app name.
|
|
*/}}
|
|
{{- define "openshell.fullname" -}}
|
|
{{- if .Values.fullnameOverride }}
|
|
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
|
{{- else }}
|
|
{{- $name := default .Chart.Name .Values.nameOverride }}
|
|
{{- if contains $name .Release.Name }}
|
|
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
|
{{- else }}
|
|
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Create chart name and version as used by the chart label.
|
|
*/}}
|
|
{{- define "openshell.chart" -}}
|
|
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Common labels
|
|
*/}}
|
|
{{- define "openshell.labels" -}}
|
|
helm.sh/chart: {{ include "openshell.chart" . }}
|
|
{{ include "openshell.selectorLabels" . }}
|
|
{{- if .Chart.AppVersion }}
|
|
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
|
{{- end }}
|
|
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Selector labels
|
|
*/}}
|
|
{{- define "openshell.selectorLabels" -}}
|
|
app.kubernetes.io/name: {{ include "openshell.name" . }}
|
|
app.kubernetes.io/instance: {{ .Release.Name }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Create the name of the service account to use
|
|
*/}}
|
|
{{- define "openshell.serviceAccountName" -}}
|
|
{{- if .Values.serviceAccount.create }}
|
|
{{- default (include "openshell.fullname" .) .Values.serviceAccount.name }}
|
|
{{- else }}
|
|
{{- default "default" .Values.serviceAccount.name }}
|
|
{{- end }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Create the name of the service account assigned to sandbox pods
|
|
*/}}
|
|
{{- define "openshell.sandboxServiceAccountName" -}}
|
|
{{- if .Values.sandboxServiceAccount.create }}
|
|
{{- default (printf "%s-sandbox" (include "openshell.fullname" .) | trunc 63 | trimSuffix "-") .Values.sandboxServiceAccount.name }}
|
|
{{- else }}
|
|
{{- default "default" .Values.sandboxServiceAccount.name }}
|
|
{{- end }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Whether this chart owns workspace-scoped resources. Missing legacy values
|
|
default to enabled so upgrades with --reuse-values preserve the old topology.
|
|
*/}}
|
|
{{- define "openshell.workspaceResourcesEnabled" -}}
|
|
{{- $workspaceResources := .Values.workspaceResources | default dict -}}
|
|
{{- $enabled := true -}}
|
|
{{- if hasKey $workspaceResources "enabled" -}}
|
|
{{- $enabled = get $workspaceResources "enabled" -}}
|
|
{{- end -}}
|
|
{{- if $enabled -}}true{{- end -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Gateway image reference. Uses image.tag when set; falls back to .Chart.AppVersion
|
|
so a released chart automatically pulls the matching image without extra overrides.
|
|
*/}}
|
|
{{- define "openshell.image" -}}
|
|
{{- printf "%s:%s" .Values.image.repository (.Values.image.tag | default .Chart.AppVersion) }}
|
|
{{- end }}
|
|
|
|
{{/* Official sandbox runtime repository used by the gateway's built-in default. */}}
|
|
{{- define "openshell.defaultSandboxRuntimeRepository" -}}
|
|
ghcr.io/nvidia/openshell/sandbox
|
|
{{- end }}
|
|
|
|
{{/* Whether Helm must propagate a sandbox runtime image override. */}}
|
|
{{- define "openshell.sandboxRuntimeImageOverrideEnabled" -}}
|
|
{{- $defaultRepository := include "openshell.defaultSandboxRuntimeRepository" . -}}
|
|
{{- $repository := .Values.sandboxRuntime.image.repository | default $defaultRepository -}}
|
|
{{- if or (ne $repository $defaultRepository) .Values.sandboxRuntime.image.tag -}}true{{- end -}}
|
|
{{- end }}
|
|
|
|
{{/* Sandbox runtime image override. */}}
|
|
{{- define "openshell.sandboxRuntimeImage" -}}
|
|
{{- $repository := .Values.sandboxRuntime.image.repository | default (include "openshell.defaultSandboxRuntimeRepository" .) -}}
|
|
{{- $tag := .Values.sandboxRuntime.image.tag | default .Values.image.tag | default .Chart.AppVersion -}}
|
|
{{- printf "%s:%s" $repository $tag }}
|
|
{{- end }}
|
|
|
|
{{/* Official supervisor repository used by the gateway's built-in default. */}}
|
|
{{- define "openshell.defaultSupervisorRepository" -}}
|
|
ghcr.io/nvidia/openshell/supervisor
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Whether the gateway listener should verify client certificates (mTLS).
|
|
An explicit empty server.tls.clientCaSecretName disables client-CA wiring in
|
|
both gateway.toml and the workload, overriding built-in PKI and cert-manager
|
|
defaults.
|
|
*/}}
|
|
{{- define "openshell.gatewayClientCaEnabled" -}}
|
|
{{- if .Values.server.disableTls -}}
|
|
{{- else if not .Values.server.tls.enableMtls -}}
|
|
{{- else if eq .Values.server.tls.clientCaSecretName "" -}}
|
|
{{- else if or .Values.server.tls.clientCaSecretName (and .Values.pkiInitJob.enabled (not .Values.certManager.enabled)) (and .Values.certManager.enabled .Values.certManager.clientCaFromServerTlsSecret) -}}
|
|
true
|
|
{{- end -}}
|
|
{{- end -}}
|
|
|
|
{{/*
|
|
Whether Helm must propagate a supervisor image override into gateway.toml.
|
|
The chart's documented repository and empty tag are the gateway-owned default.
|
|
*/}}
|
|
{{- define "openshell.supervisorImageOverrideEnabled" -}}
|
|
{{- $defaultRepository := include "openshell.defaultSupervisorRepository" . -}}
|
|
{{- $repository := .Values.supervisor.image.repository | default $defaultRepository -}}
|
|
{{- if or (ne $repository $defaultRepository) .Values.supervisor.image.tag -}}true{{- end -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Supervisor image override. A tag-only override uses the official repository;
|
|
a repository-only override uses the effective gateway image tag.
|
|
*/}}
|
|
{{- define "openshell.supervisorImage" -}}
|
|
{{- $repository := .Values.supervisor.image.repository | default (include "openshell.defaultSupervisorRepository" .) -}}
|
|
{{- $tag := .Values.supervisor.image.tag | default .Values.image.tag | default .Chart.AppVersion -}}
|
|
{{- printf "%s:%s" $repository $tag }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Namespaced Issuer (selfSigned) for cert-manager CA bootstrap.
|
|
*/}}
|
|
{{- define "openshell.issuerSelfSigned" -}}
|
|
{{- printf "%s-selfsigned" (include "openshell.fullname" .) | trunc 63 | trimSuffix "-" }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Namespace where sandbox pods are created. An explicit
|
|
.Values.server.sandboxNamespace is used verbatim. Otherwise it defaults to
|
|
.Release.Namespace so `helm install -n my-ns` works without extra overrides.
|
|
*/}}
|
|
{{- define "openshell.sandboxNamespace" -}}
|
|
{{- .Values.server.sandboxNamespace | default .Release.Namespace -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Namespace where Kubernetes Secret-backed provider credentials live.
|
|
*/}}
|
|
{{- define "openshell.credentialKubernetesSecretsNamespace" -}}
|
|
{{- .Values.server.credentialDrivers.kubernetesSecrets.namespace | default .Release.Namespace -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Name of the Secret holding the default credential storage key-encryption key.
|
|
When server.credentialStorage.existingSecret is set, returns that name instead
|
|
of the chart-generated name (for GitOps / helm-template workflows).
|
|
*/}}
|
|
{{- define "openshell.credentialStorageKeyEncryptionKeySecretName" -}}
|
|
{{- if .Values.server.credentialStorage.existingSecret -}}
|
|
{{- .Values.server.credentialStorage.existingSecret -}}
|
|
{{- else -}}
|
|
{{- printf "%s-credential-storage-key-encryption-key" (include "openshell.fullname" .) | trunc 63 | trimSuffix "-" -}}
|
|
{{- end -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Key inside the default credential storage key-encryption key Secret.
|
|
*/}}
|
|
{{- define "openshell.credentialStorageKeyEncryptionKeySecretKey" -}}
|
|
key-encryption-key
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Gateway environment variable used to pass the default credential storage key-encryption key.
|
|
*/}}
|
|
{{- define "openshell.credentialStorageKeyEncryptionKeyEnvName" -}}
|
|
OPENSHELL_GATEWAY_CREDENTIAL_KEY_ENCRYPTION_KEY
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Name of the Secret holding gateway-minted sandbox JWT signing material.
|
|
*/}}
|
|
{{- define "openshell.sandboxJwtSecretName" -}}
|
|
{{- .Values.server.sandboxJwt.signingSecretName | default (printf "%s-jwt-keys" (include "openshell.fullname" .)) -}}
|
|
{{- end }}
|
|
|
|
{{- define "openshell.peerServiceName" -}}
|
|
{{- printf "%s-peer" (include "openshell.fullname" .) -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
gRPC endpoint sandbox pods use to call back into the gateway. An explicit
|
|
.Values.server.grpcEndpoint is used verbatim. Otherwise it is derived from
|
|
the in-cluster Service DNS, release namespace, service port, and disableTls
|
|
flag — so the default value works for any release name or namespace without
|
|
override.
|
|
*/}}
|
|
{{- define "openshell.grpcEndpoint" -}}
|
|
{{- if .Values.server.grpcEndpoint -}}
|
|
{{- .Values.server.grpcEndpoint -}}
|
|
{{- else -}}
|
|
{{- $scheme := ternary "http" "https" (default false .Values.server.disableTls) -}}
|
|
{{- printf "%s://%s.%s.svc.cluster.local:%d" $scheme (include "openshell.fullname" .) .Release.Namespace (int .Values.service.port) -}}
|
|
{{- end -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Default server certificate DNS SANs derived from the release name and namespace.
|
|
Returns a YAML list. Append extra SANs from values with range loops.
|
|
*/}}
|
|
{{- define "openshell.defaultServerDnsNames" -}}
|
|
{{- $name := include "openshell.fullname" . -}}
|
|
{{- $ns := .Release.Namespace -}}
|
|
{{- list $name
|
|
(printf "%s.%s.svc" $name $ns)
|
|
(printf "%s.%s.svc.cluster.local" $name $ns)
|
|
"localhost"
|
|
(printf "%s.localhost" $name)
|
|
(printf "*.%s.localhost" $name)
|
|
"host.docker.internal"
|
|
"host.containers.internal"
|
|
| toYaml }}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Name of the ConfigMap holding the backend CA for BackendTLSPolicy validation.
|
|
*/}}
|
|
{{- define "openshell.backendCaConfigMapName" -}}
|
|
{{- .Values.grpcRoute.backendTLSPolicy.caCertificateConfigMapName | default (printf "%s-backend-ca" (include "openshell.fullname" .)) -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Gateway workload kind. StatefulSet is the default because the default SQLite
|
|
database requires persistent per-pod storage.
|
|
*/}}
|
|
{{- define "openshell.workloadKind" -}}
|
|
{{- $workload := .Values.workload | default dict -}}
|
|
{{- if not (kindIs "map" $workload) -}}
|
|
{{- fail "workload must be a map with kind and allowMultiReplicaStatefulSet fields." -}}
|
|
{{- end -}}
|
|
{{- default "statefulset" (get $workload "kind") | lower -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Translate chart image pull policy values to the canonical gateway vocabulary.
|
|
The Kubernetes spellings remain accepted so existing values files continue to
|
|
work across the schema-v2 chart upgrade.
|
|
*/}}
|
|
{{- define "openshell.canonicalImagePullPolicy" -}}
|
|
{{- $policy := printf "%v" . -}}
|
|
{{- if eq $policy "Always" -}}
|
|
always
|
|
{{- else if eq $policy "IfNotPresent" -}}
|
|
if_not_present
|
|
{{- else if eq $policy "Never" -}}
|
|
never
|
|
{{- else if has $policy (list "always" "if_not_present" "never") -}}
|
|
{{- $policy -}}
|
|
{{- else -}}
|
|
{{- fail (printf "image pull policy %q must be one of: always, if_not_present, never, Always, IfNotPresent, Never" $policy) -}}
|
|
{{- end -}}
|
|
{{- end }}
|
|
|
|
{{/*
|
|
Validate chart values that Helm would otherwise accept silently.
|
|
*/}}
|
|
{{- define "openshell.validateValues" -}}
|
|
{{- $workloadKind := include "openshell.workloadKind" . -}}
|
|
{{- $workload := .Values.workload | default dict -}}
|
|
{{- $replicaCount := int (default 1 .Values.replicaCount) -}}
|
|
{{- if and (hasKey .Values "postgres") (kindIs "map" .Values.postgres) (hasKey .Values.postgres "enabled") -}}
|
|
{{- fail "postgres.enabled was removed; the OpenShell chart no longer deploys PostgreSQL. Provision PostgreSQL separately and set server.externalDbSecret to a Secret containing a PostgreSQL URI." -}}
|
|
{{- end -}}
|
|
{{- if not (or (eq $workloadKind "statefulset") (eq $workloadKind "deployment")) -}}
|
|
{{- fail "workload.kind must be one of: statefulset, deployment." -}}
|
|
{{- end -}}
|
|
{{- if and (eq $workloadKind "deployment") (not .Values.server.externalDbSecret) -}}
|
|
{{- fail "workload.kind=deployment requires server.externalDbSecret; use workload.kind=statefulset for the default SQLite database." -}}
|
|
{{- end -}}
|
|
{{- if and (gt $replicaCount 1) (not .Values.server.externalDbSecret) -}}
|
|
{{- fail "replicaCount > 1 requires server.externalDbSecret; multiple gateway replicas cannot share the default per-pod SQLite database." -}}
|
|
{{- end -}}
|
|
{{- if and (eq $workloadKind "statefulset") (gt $replicaCount 1) (not (get $workload "allowMultiReplicaStatefulSet" | default false)) -}}
|
|
{{- fail "replicaCount > 1 with workload.kind=statefulset requires workload.allowMultiReplicaStatefulSet=true; use workload.kind=deployment for external database-backed multi-replica gateways." -}}
|
|
{{- end -}}
|
|
{{- $workspaceMode := .Values.server.drivers.kubernetes.workspaceMode | default "shared" -}}
|
|
{{- if not (has $workspaceMode (list "shared" "managed" "operator")) -}}
|
|
{{- fail "server.drivers.kubernetes.workspaceMode must be one of: shared, managed, operator." -}}
|
|
{{- end -}}
|
|
{{- $credentialDrivers := list -}}
|
|
{{- if .Values.server.credentialDrivers.kubernetesSecrets.enabled -}}
|
|
{{- $credentialDrivers = append $credentialDrivers "kubernetes-secrets" -}}
|
|
{{- end -}}
|
|
{{- if .Values.server.credentialDrivers.vault.enabled -}}
|
|
{{- $credentialDrivers = append $credentialDrivers "vault" -}}
|
|
{{- end -}}
|
|
{{- if gt (len $credentialDrivers) 1 -}}
|
|
{{- fail "only one external server.credentialDrivers backend can be enabled at a time." -}}
|
|
{{- end -}}
|
|
{{- if kindIs "invalid" .Values.server.tls.clientCaSecretName -}}
|
|
{{- fail "server.tls.clientCaSecretName cannot be null; omit the key to use the chart default (openshell-server-client-ca), or set to \"\" to disable client certificate verification for HTTPS-only mode" -}}
|
|
{{- end -}}
|
|
{{- end }}
|